projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'regmap/fix/debugfs' into tmp
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2 *
3 * Copyright (c) 2007-2012 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 *
9 * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10 * Written by: Eliezer Tamir
11 * Based on code from Michael Chan's bnx2 driver
12 * UDP CSUM errata workaround by Arik Gendelman
13 * Slowpath and fastpath rework by Vladislav Zolotarov
14 * Statistics and Link management by Yitchak Gertner
15 *
16 */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/ipv6.h>
25 #include <net/ip6_checksum.h>
26 #include <linux/prefetch.h>
27 #include "bnx2x_cmn.h"
28 #include "bnx2x_init.h"
29 #include "bnx2x_sp.h"
30
31
32
33 /**
34 * bnx2x_move_fp - move content of the fastpath structure.
35 *
36 * @bp: driver handle
37 * @from: source FP index
38 * @to: destination FP index
39 *
40 * Makes sure the contents of the bp->fp[to].napi is kept
41 * intact. This is done by first copying the napi struct from
42 * the target to the source, and then mem copying the entire
43 * source onto the target. Update txdata pointers and related
44 * content.
45 */
46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
47 {
48 struct bnx2x_fastpath *from_fp = &bp->fp[from];
49 struct bnx2x_fastpath *to_fp = &bp->fp[to];
50 struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
51 struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
52 struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
53 struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
54 int old_max_eth_txqs, new_max_eth_txqs;
55 int old_txdata_index = 0, new_txdata_index = 0;
56
57 /* Copy the NAPI object as it has been already initialized */
58 from_fp->napi = to_fp->napi;
59
60 /* Move bnx2x_fastpath contents */
61 memcpy(to_fp, from_fp, sizeof(*to_fp));
62 to_fp->index = to;
63
64 /* move sp_objs contents as well, as their indices match fp ones */
65 memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
66
67 /* move fp_stats contents as well, as their indices match fp ones */
68 memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
69
70 /* Update txdata pointers in fp and move txdata content accordingly:
71 * Each fp consumes 'max_cos' txdata structures, so the index should be
72 * decremented by max_cos x delta.
73 */
74
75 old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
76 new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
77 (bp)->max_cos;
78 if (from == FCOE_IDX(bp)) {
79 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
80 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
81 }
82
83 memcpy(&bp->bnx2x_txq[new_txdata_index],
84 &bp->bnx2x_txq[old_txdata_index],
85 sizeof(struct bnx2x_fp_txdata));
86 to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
87 }
88
89 /**
90 * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
91 *
92 * @bp: driver handle
93 * @delta: number of eth queues which were not allocated
94 */
95 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
96 {
97 int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
98
99 /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
100 * backward along the array could cause memory to be overriden
101 */
102 for (cos = 1; cos < bp->max_cos; cos++) {
103 for (i = 0; i < old_eth_num - delta; i++) {
104 struct bnx2x_fastpath *fp = &bp->fp[i];
105 int new_idx = cos * (old_eth_num - delta) + i;
106
107 memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
108 sizeof(struct bnx2x_fp_txdata));
109 fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
110 }
111 }
112 }
113
114 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
115
116 /* free skb in the packet ring at pos idx
117 * return idx of last bd freed
118 */
119 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
120 u16 idx, unsigned int *pkts_compl,
121 unsigned int *bytes_compl)
122 {
123 struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
124 struct eth_tx_start_bd *tx_start_bd;
125 struct eth_tx_bd *tx_data_bd;
126 struct sk_buff *skb = tx_buf->skb;
127 u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
128 int nbd;
129
130 /* prefetch skb end pointer to speedup dev_kfree_skb() */
131 prefetch(&skb->end);
132
133 DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d buff @(%p)->skb %p\n",
134 txdata->txq_index, idx, tx_buf, skb);
135
136 /* unmap first bd */
137 tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
138 dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
139 BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
140
141
142 nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
143 #ifdef BNX2X_STOP_ON_ERROR
144 if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
145 BNX2X_ERR("BAD nbd!\n");
146 bnx2x_panic();
147 }
148 #endif
149 new_cons = nbd + tx_buf->first_bd;
150
151 /* Get the next bd */
152 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
153
154 /* Skip a parse bd... */
155 --nbd;
156 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
157
158 /* ...and the TSO split header bd since they have no mapping */
159 if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
160 --nbd;
161 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
162 }
163
164 /* now free frags */
165 while (nbd > 0) {
166
167 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
168 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
169 BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
170 if (--nbd)
171 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
172 }
173
174 /* release skb */
175 WARN_ON(!skb);
176 if (likely(skb)) {
177 (*pkts_compl)++;
178 (*bytes_compl) += skb->len;
179 }
180
181 dev_kfree_skb_any(skb);
182 tx_buf->first_bd = 0;
183 tx_buf->skb = NULL;
184
185 return new_cons;
186 }
187
188 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
189 {
190 struct netdev_queue *txq;
191 u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
192 unsigned int pkts_compl = 0, bytes_compl = 0;
193
194 #ifdef BNX2X_STOP_ON_ERROR
195 if (unlikely(bp->panic))
196 return -1;
197 #endif
198
199 txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
200 hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
201 sw_cons = txdata->tx_pkt_cons;
202
203 while (sw_cons != hw_cons) {
204 u16 pkt_cons;
205
206 pkt_cons = TX_BD(sw_cons);
207
208 DP(NETIF_MSG_TX_DONE,
209 "queue[%d]: hw_cons %u sw_cons %u pkt_cons %u\n",
210 txdata->txq_index, hw_cons, sw_cons, pkt_cons);
211
212 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
213 &pkts_compl, &bytes_compl);
214
215 sw_cons++;
216 }
217
218 netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
219
220 txdata->tx_pkt_cons = sw_cons;
221 txdata->tx_bd_cons = bd_cons;
222
223 /* Need to make the tx_bd_cons update visible to start_xmit()
224 * before checking for netif_tx_queue_stopped(). Without the
225 * memory barrier, there is a small possibility that
226 * start_xmit() will miss it and cause the queue to be stopped
227 * forever.
228 * On the other hand we need an rmb() here to ensure the proper
229 * ordering of bit testing in the following
230 * netif_tx_queue_stopped(txq) call.
231 */
232 smp_mb();
233
234 if (unlikely(netif_tx_queue_stopped(txq))) {
235 /* Taking tx_lock() is needed to prevent reenabling the queue
236 * while it's empty. This could have happen if rx_action() gets
237 * suspended in bnx2x_tx_int() after the condition before
238 * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
239 *
240 * stops the queue->sees fresh tx_bd_cons->releases the queue->
241 * sends some packets consuming the whole queue again->
242 * stops the queue
243 */
244
245 __netif_tx_lock(txq, smp_processor_id());
246
247 if ((netif_tx_queue_stopped(txq)) &&
248 (bp->state == BNX2X_STATE_OPEN) &&
249 (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
250 netif_tx_wake_queue(txq);
251
252 __netif_tx_unlock(txq);
253 }
254 return 0;
255 }
256
257 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
258 u16 idx)
259 {
260 u16 last_max = fp->last_max_sge;
261
262 if (SUB_S16(idx, last_max) > 0)
263 fp->last_max_sge = idx;
264 }
265
266 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
267 u16 sge_len,
268 struct eth_end_agg_rx_cqe *cqe)
269 {
270 struct bnx2x *bp = fp->bp;
271 u16 last_max, last_elem, first_elem;
272 u16 delta = 0;
273 u16 i;
274
275 if (!sge_len)
276 return;
277
278 /* First mark all used pages */
279 for (i = 0; i < sge_len; i++)
280 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
281 RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
282
283 DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
284 sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
285
286 /* Here we assume that the last SGE index is the biggest */
287 prefetch((void *)(fp->sge_mask));
288 bnx2x_update_last_max_sge(fp,
289 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
290
291 last_max = RX_SGE(fp->last_max_sge);
292 last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
293 first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
294
295 /* If ring is not full */
296 if (last_elem + 1 != first_elem)
297 last_elem++;
298
299 /* Now update the prod */
300 for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
301 if (likely(fp->sge_mask[i]))
302 break;
303
304 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
305 delta += BIT_VEC64_ELEM_SZ;
306 }
307
308 if (delta > 0) {
309 fp->rx_sge_prod += delta;
310 /* clear page-end entries */
311 bnx2x_clear_sge_mask_next_elems(fp);
312 }
313
314 DP(NETIF_MSG_RX_STATUS,
315 "fp->last_max_sge = %d fp->rx_sge_prod = %d\n",
316 fp->last_max_sge, fp->rx_sge_prod);
317 }
318
319 /* Set Toeplitz hash value in the skb using the value from the
320 * CQE (calculated by HW).
321 */
322 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
323 const struct eth_fast_path_rx_cqe *cqe,
324 bool *l4_rxhash)
325 {
326 /* Set Toeplitz hash from CQE */
327 if ((bp->dev->features & NETIF_F_RXHASH) &&
328 (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
329 enum eth_rss_hash_type htype;
330
331 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
332 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
333 (htype == TCP_IPV6_HASH_TYPE);
334 return le32_to_cpu(cqe->rss_hash_result);
335 }
336 *l4_rxhash = false;
337 return 0;
338 }
339
340 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
341 u16 cons, u16 prod,
342 struct eth_fast_path_rx_cqe *cqe)
343 {
344 struct bnx2x *bp = fp->bp;
345 struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
346 struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
347 struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
348 dma_addr_t mapping;
349 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
350 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
351
352 /* print error if current state != stop */
353 if (tpa_info->tpa_state != BNX2X_TPA_STOP)
354 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
355
356 /* Try to map an empty data buffer from the aggregation info */
357 mapping = dma_map_single(&bp->pdev->dev,
358 first_buf->data + NET_SKB_PAD,
359 fp->rx_buf_size, DMA_FROM_DEVICE);
360 /*
361 * ...if it fails - move the skb from the consumer to the producer
362 * and set the current aggregation state as ERROR to drop it
363 * when TPA_STOP arrives.
364 */
365
366 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
367 /* Move the BD from the consumer to the producer */
368 bnx2x_reuse_rx_data(fp, cons, prod);
369 tpa_info->tpa_state = BNX2X_TPA_ERROR;
370 return;
371 }
372
373 /* move empty data from pool to prod */
374 prod_rx_buf->data = first_buf->data;
375 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
376 /* point prod_bd to new data */
377 prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
378 prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
379
380 /* move partial skb from cons to pool (don't unmap yet) */
381 *first_buf = *cons_rx_buf;
382
383 /* mark bin state as START */
384 tpa_info->parsing_flags =
385 le16_to_cpu(cqe->pars_flags.flags);
386 tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
387 tpa_info->tpa_state = BNX2X_TPA_START;
388 tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
389 tpa_info->placement_offset = cqe->placement_offset;
390 tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
391 if (fp->mode == TPA_MODE_GRO) {
392 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
393 tpa_info->full_page =
394 SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
395 tpa_info->gro_size = gro_size;
396 }
397
398 #ifdef BNX2X_STOP_ON_ERROR
399 fp->tpa_queue_used |= (1 << queue);
400 #ifdef _ASM_GENERIC_INT_L64_H
401 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
402 #else
403 DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
404 #endif
405 fp->tpa_queue_used);
406 #endif
407 }
408
409 /* Timestamp option length allowed for TPA aggregation:
410 *
411 * nop nop kind length echo val
412 */
413 #define TPA_TSTAMP_OPT_LEN 12
414 /**
415 * bnx2x_set_lro_mss - calculate the approximate value of the MSS
416 *
417 * @bp: driver handle
418 * @parsing_flags: parsing flags from the START CQE
419 * @len_on_bd: total length of the first packet for the
420 * aggregation.
421 *
422 * Approximate value of the MSS for this aggregation calculated using
423 * the first packet of it.
424 */
425 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
426 u16 len_on_bd)
427 {
428 /*
429 * TPA arrgregation won't have either IP options or TCP options
430 * other than timestamp or IPv6 extension headers.
431 */
432 u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
433
434 if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
435 PRS_FLAG_OVERETH_IPV6)
436 hdrs_len += sizeof(struct ipv6hdr);
437 else /* IPv4 */
438 hdrs_len += sizeof(struct iphdr);
439
440
441 /* Check if there was a TCP timestamp, if there is it's will
442 * always be 12 bytes length: nop nop kind length echo val.
443 *
444 * Otherwise FW would close the aggregation.
445 */
446 if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
447 hdrs_len += TPA_TSTAMP_OPT_LEN;
448
449 return len_on_bd - hdrs_len;
450 }
451
452 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
453 struct bnx2x_fastpath *fp, u16 index)
454 {
455 struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
456 struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
457 struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
458 dma_addr_t mapping;
459
460 if (unlikely(page == NULL)) {
461 BNX2X_ERR("Can't alloc sge\n");
462 return -ENOMEM;
463 }
464
465 mapping = dma_map_page(&bp->pdev->dev, page, 0,
466 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
467 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
468 __free_pages(page, PAGES_PER_SGE_SHIFT);
469 BNX2X_ERR("Can't map sge\n");
470 return -ENOMEM;
471 }
472
473 sw_buf->page = page;
474 dma_unmap_addr_set(sw_buf, mapping, mapping);
475
476 sge->addr_hi = cpu_to_le32(U64_HI(mapping));
477 sge->addr_lo = cpu_to_le32(U64_LO(mapping));
478
479 return 0;
480 }
481
482 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
483 struct bnx2x_agg_info *tpa_info,
484 u16 pages,
485 struct sk_buff *skb,
486 struct eth_end_agg_rx_cqe *cqe,
487 u16 cqe_idx)
488 {
489 struct sw_rx_page *rx_pg, old_rx_pg;
490 u32 i, frag_len, frag_size;
491 int err, j, frag_id = 0;
492 u16 len_on_bd = tpa_info->len_on_bd;
493 u16 full_page = 0, gro_size = 0;
494
495 frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
496
497 if (fp->mode == TPA_MODE_GRO) {
498 gro_size = tpa_info->gro_size;
499 full_page = tpa_info->full_page;
500 }
501
502 /* This is needed in order to enable forwarding support */
503 if (frag_size) {
504 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
505 tpa_info->parsing_flags, len_on_bd);
506
507 /* set for GRO */
508 if (fp->mode == TPA_MODE_GRO)
509 skb_shinfo(skb)->gso_type =
510 (GET_FLAG(tpa_info->parsing_flags,
511 PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
512 PRS_FLAG_OVERETH_IPV6) ?
513 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
514 }
515
516
517 #ifdef BNX2X_STOP_ON_ERROR
518 if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
519 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
520 pages, cqe_idx);
521 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
522 bnx2x_panic();
523 return -EINVAL;
524 }
525 #endif
526
527 /* Run through the SGL and compose the fragmented skb */
528 for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
529 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
530
531 /* FW gives the indices of the SGE as if the ring is an array
532 (meaning that "next" element will consume 2 indices) */
533 if (fp->mode == TPA_MODE_GRO)
534 frag_len = min_t(u32, frag_size, (u32)full_page);
535 else /* LRO */
536 frag_len = min_t(u32, frag_size,
537 (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
538
539 rx_pg = &fp->rx_page_ring[sge_idx];
540 old_rx_pg = *rx_pg;
541
542 /* If we fail to allocate a substitute page, we simply stop
543 where we are and drop the whole packet */
544 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
545 if (unlikely(err)) {
546 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
547 return err;
548 }
549
550 /* Unmap the page as we r going to pass it to the stack */
551 dma_unmap_page(&bp->pdev->dev,
552 dma_unmap_addr(&old_rx_pg, mapping),
553 SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
554 /* Add one frag and update the appropriate fields in the skb */
555 if (fp->mode == TPA_MODE_LRO)
556 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
557 else { /* GRO */
558 int rem;
559 int offset = 0;
560 for (rem = frag_len; rem > 0; rem -= gro_size) {
561 int len = rem > gro_size ? gro_size : rem;
562 skb_fill_page_desc(skb, frag_id++,
563 old_rx_pg.page, offset, len);
564 if (offset)
565 get_page(old_rx_pg.page);
566 offset += len;
567 }
568 }
569
570 skb->data_len += frag_len;
571 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
572 skb->len += frag_len;
573
574 frag_size -= frag_len;
575 }
576
577 return 0;
578 }
579
580 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
581 {
582 if (fp->rx_frag_size)
583 put_page(virt_to_head_page(data));
584 else
585 kfree(data);
586 }
587
588 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
589 {
590 if (fp->rx_frag_size)
591 return netdev_alloc_frag(fp->rx_frag_size);
592
593 return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
594 }
595
596
597 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
598 struct bnx2x_agg_info *tpa_info,
599 u16 pages,
600 struct eth_end_agg_rx_cqe *cqe,
601 u16 cqe_idx)
602 {
603 struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
604 u8 pad = tpa_info->placement_offset;
605 u16 len = tpa_info->len_on_bd;
606 struct sk_buff *skb = NULL;
607 u8 *new_data, *data = rx_buf->data;
608 u8 old_tpa_state = tpa_info->tpa_state;
609
610 tpa_info->tpa_state = BNX2X_TPA_STOP;
611
612 /* If we there was an error during the handling of the TPA_START -
613 * drop this aggregation.
614 */
615 if (old_tpa_state == BNX2X_TPA_ERROR)
616 goto drop;
617
618 /* Try to allocate the new data */
619 new_data = bnx2x_frag_alloc(fp);
620 /* Unmap skb in the pool anyway, as we are going to change
621 pool entry status to BNX2X_TPA_STOP even if new skb allocation
622 fails. */
623 dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
624 fp->rx_buf_size, DMA_FROM_DEVICE);
625 if (likely(new_data))
626 skb = build_skb(data, fp->rx_frag_size);
627
628 if (likely(skb)) {
629 #ifdef BNX2X_STOP_ON_ERROR
630 if (pad + len > fp->rx_buf_size) {
631 BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n",
632 pad, len, fp->rx_buf_size);
633 bnx2x_panic();
634 return;
635 }
636 #endif
637
638 skb_reserve(skb, pad + NET_SKB_PAD);
639 skb_put(skb, len);
640 skb->rxhash = tpa_info->rxhash;
641 skb->l4_rxhash = tpa_info->l4_rxhash;
642
643 skb->protocol = eth_type_trans(skb, bp->dev);
644 skb->ip_summed = CHECKSUM_UNNECESSARY;
645
646 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
647 skb, cqe, cqe_idx)) {
648 if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
649 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
650 napi_gro_receive(&fp->napi, skb);
651 } else {
652 DP(NETIF_MSG_RX_STATUS,
653 "Failed to allocate new pages - dropping packet!\n");
654 dev_kfree_skb_any(skb);
655 }
656
657
658 /* put new data in bin */
659 rx_buf->data = new_data;
660
661 return;
662 }
663 bnx2x_frag_free(fp, new_data);
664 drop:
665 /* drop the packet and keep the buffer in the bin */
666 DP(NETIF_MSG_RX_STATUS,
667 "Failed to allocate or map a new skb - dropping packet!\n");
668 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
669 }
670
671 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
672 struct bnx2x_fastpath *fp, u16 index)
673 {
674 u8 *data;
675 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
676 struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
677 dma_addr_t mapping;
678
679 data = bnx2x_frag_alloc(fp);
680 if (unlikely(data == NULL))
681 return -ENOMEM;
682
683 mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
684 fp->rx_buf_size,
685 DMA_FROM_DEVICE);
686 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
687 bnx2x_frag_free(fp, data);
688 BNX2X_ERR("Can't map rx data\n");
689 return -ENOMEM;
690 }
691
692 rx_buf->data = data;
693 dma_unmap_addr_set(rx_buf, mapping, mapping);
694
695 rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
696 rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
697
698 return 0;
699 }
700
701 static
702 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
703 struct bnx2x_fastpath *fp,
704 struct bnx2x_eth_q_stats *qstats)
705 {
706 /* Do nothing if no L4 csum validation was done.
707 * We do not check whether IP csum was validated. For IPv4 we assume
708 * that if the card got as far as validating the L4 csum, it also
709 * validated the IP csum. IPv6 has no IP csum.
710 */
711 if (cqe->fast_path_cqe.status_flags &
712 ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
713 return;
714
715 /* If L4 validation was done, check if an error was found. */
716
717 if (cqe->fast_path_cqe.type_error_flags &
718 (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
719 ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
720 qstats->hw_csum_err++;
721 else
722 skb->ip_summed = CHECKSUM_UNNECESSARY;
723 }
724
725 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
726 {
727 struct bnx2x *bp = fp->bp;
728 u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
729 u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
730 int rx_pkt = 0;
731
732 #ifdef BNX2X_STOP_ON_ERROR
733 if (unlikely(bp->panic))
734 return 0;
735 #endif
736
737 /* CQ "next element" is of the size of the regular element,
738 that's why it's ok here */
739 hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
740 if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
741 hw_comp_cons++;
742
743 bd_cons = fp->rx_bd_cons;
744 bd_prod = fp->rx_bd_prod;
745 bd_prod_fw = bd_prod;
746 sw_comp_cons = fp->rx_comp_cons;
747 sw_comp_prod = fp->rx_comp_prod;
748
749 /* Memory barrier necessary as speculative reads of the rx
750 * buffer can be ahead of the index in the status block
751 */
752 rmb();
753
754 DP(NETIF_MSG_RX_STATUS,
755 "queue[%d]: hw_comp_cons %u sw_comp_cons %u\n",
756 fp->index, hw_comp_cons, sw_comp_cons);
757
758 while (sw_comp_cons != hw_comp_cons) {
759 struct sw_rx_bd *rx_buf = NULL;
760 struct sk_buff *skb;
761 union eth_rx_cqe *cqe;
762 struct eth_fast_path_rx_cqe *cqe_fp;
763 u8 cqe_fp_flags;
764 enum eth_rx_cqe_type cqe_fp_type;
765 u16 len, pad, queue;
766 u8 *data;
767 bool l4_rxhash;
768
769 #ifdef BNX2X_STOP_ON_ERROR
770 if (unlikely(bp->panic))
771 return 0;
772 #endif
773
774 comp_ring_cons = RCQ_BD(sw_comp_cons);
775 bd_prod = RX_BD(bd_prod);
776 bd_cons = RX_BD(bd_cons);
777
778 cqe = &fp->rx_comp_ring[comp_ring_cons];
779 cqe_fp = &cqe->fast_path_cqe;
780 cqe_fp_flags = cqe_fp->type_error_flags;
781 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
782
783 DP(NETIF_MSG_RX_STATUS,
784 "CQE type %x err %x status %x queue %x vlan %x len %u\n",
785 CQE_TYPE(cqe_fp_flags),
786 cqe_fp_flags, cqe_fp->status_flags,
787 le32_to_cpu(cqe_fp->rss_hash_result),
788 le16_to_cpu(cqe_fp->vlan_tag),
789 le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
790
791 /* is this a slowpath msg? */
792 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
793 bnx2x_sp_event(fp, cqe);
794 goto next_cqe;
795 }
796
797 rx_buf = &fp->rx_buf_ring[bd_cons];
798 data = rx_buf->data;
799
800 if (!CQE_TYPE_FAST(cqe_fp_type)) {
801 struct bnx2x_agg_info *tpa_info;
802 u16 frag_size, pages;
803 #ifdef BNX2X_STOP_ON_ERROR
804 /* sanity check */
805 if (fp->disable_tpa &&
806 (CQE_TYPE_START(cqe_fp_type) ||
807 CQE_TYPE_STOP(cqe_fp_type)))
808 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
809 CQE_TYPE(cqe_fp_type));
810 #endif
811
812 if (CQE_TYPE_START(cqe_fp_type)) {
813 u16 queue = cqe_fp->queue_index;
814 DP(NETIF_MSG_RX_STATUS,
815 "calling tpa_start on queue %d\n",
816 queue);
817
818 bnx2x_tpa_start(fp, queue,
819 bd_cons, bd_prod,
820 cqe_fp);
821
822 goto next_rx;
823
824 }
825 queue = cqe->end_agg_cqe.queue_index;
826 tpa_info = &fp->tpa_info[queue];
827 DP(NETIF_MSG_RX_STATUS,
828 "calling tpa_stop on queue %d\n",
829 queue);
830
831 frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
832 tpa_info->len_on_bd;
833
834 if (fp->mode == TPA_MODE_GRO)
835 pages = (frag_size + tpa_info->full_page - 1) /
836 tpa_info->full_page;
837 else
838 pages = SGE_PAGE_ALIGN(frag_size) >>
839 SGE_PAGE_SHIFT;
840
841 bnx2x_tpa_stop(bp, fp, tpa_info, pages,
842 &cqe->end_agg_cqe, comp_ring_cons);
843 #ifdef BNX2X_STOP_ON_ERROR
844 if (bp->panic)
845 return 0;
846 #endif
847
848 bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
849 goto next_cqe;
850 }
851 /* non TPA */
852 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
853 pad = cqe_fp->placement_offset;
854 dma_sync_single_for_cpu(&bp->pdev->dev,
855 dma_unmap_addr(rx_buf, mapping),
856 pad + RX_COPY_THRESH,
857 DMA_FROM_DEVICE);
858 pad += NET_SKB_PAD;
859 prefetch(data + pad); /* speedup eth_type_trans() */
860 /* is this an error packet? */
861 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
862 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
863 "ERROR flags %x rx packet %u\n",
864 cqe_fp_flags, sw_comp_cons);
865 bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
866 goto reuse_rx;
867 }
868
869 /* Since we don't have a jumbo ring
870 * copy small packets if mtu > 1500
871 */
872 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
873 (len <= RX_COPY_THRESH)) {
874 skb = netdev_alloc_skb_ip_align(bp->dev, len);
875 if (skb == NULL) {
876 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
877 "ERROR packet dropped because of alloc failure\n");
878 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
879 goto reuse_rx;
880 }
881 memcpy(skb->data, data + pad, len);
882 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
883 } else {
884 if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
885 dma_unmap_single(&bp->pdev->dev,
886 dma_unmap_addr(rx_buf, mapping),
887 fp->rx_buf_size,
888 DMA_FROM_DEVICE);
889 skb = build_skb(data, fp->rx_frag_size);
890 if (unlikely(!skb)) {
891 bnx2x_frag_free(fp, data);
892 bnx2x_fp_qstats(bp, fp)->
893 rx_skb_alloc_failed++;
894 goto next_rx;
895 }
896 skb_reserve(skb, pad);
897 } else {
898 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
899 "ERROR packet dropped because of alloc failure\n");
900 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
901 reuse_rx:
902 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
903 goto next_rx;
904 }
905 }
906
907 skb_put(skb, len);
908 skb->protocol = eth_type_trans(skb, bp->dev);
909
910 /* Set Toeplitz hash for a none-LRO skb */
911 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
912 skb->l4_rxhash = l4_rxhash;
913
914 skb_checksum_none_assert(skb);
915
916 if (bp->dev->features & NETIF_F_RXCSUM)
917 bnx2x_csum_validate(skb, cqe, fp,
918 bnx2x_fp_qstats(bp, fp));
919
920 skb_record_rx_queue(skb, fp->rx_queue);
921
922 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
923 PARSING_FLAGS_VLAN)
924 __vlan_hwaccel_put_tag(skb,
925 le16_to_cpu(cqe_fp->vlan_tag));
926 napi_gro_receive(&fp->napi, skb);
927
928
929 next_rx:
930 rx_buf->data = NULL;
931
932 bd_cons = NEXT_RX_IDX(bd_cons);
933 bd_prod = NEXT_RX_IDX(bd_prod);
934 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
935 rx_pkt++;
936 next_cqe:
937 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
938 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
939
940 if (rx_pkt == budget)
941 break;
942 } /* while */
943
944 fp->rx_bd_cons = bd_cons;
945 fp->rx_bd_prod = bd_prod_fw;
946 fp->rx_comp_cons = sw_comp_cons;
947 fp->rx_comp_prod = sw_comp_prod;
948
949 /* Update producers */
950 bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
951 fp->rx_sge_prod);
952
953 fp->rx_pkt += rx_pkt;
954 fp->rx_calls++;
955
956 return rx_pkt;
957 }
958
959 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
960 {
961 struct bnx2x_fastpath *fp = fp_cookie;
962 struct bnx2x *bp = fp->bp;
963 u8 cos;
964
965 DP(NETIF_MSG_INTR,
966 "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
967 fp->index, fp->fw_sb_id, fp->igu_sb_id);
968 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
969
970 #ifdef BNX2X_STOP_ON_ERROR
971 if (unlikely(bp->panic))
972 return IRQ_HANDLED;
973 #endif
974
975 /* Handle Rx and Tx according to MSI-X vector */
976 prefetch(fp->rx_cons_sb);
977
978 for_each_cos_in_tx_queue(fp, cos)
979 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
980
981 prefetch(&fp->sb_running_index[SM_RX_ID]);
982 napi_schedule(&bnx2x_fp(bp, fp->index, napi));
983
984 return IRQ_HANDLED;
985 }
986
987 /* HW Lock for shared dual port PHYs */
988 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
989 {
990 mutex_lock(&bp->port.phy_mutex);
991
992 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
993 }
994
995 void bnx2x_release_phy_lock(struct bnx2x *bp)
996 {
997 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
998
999 mutex_unlock(&bp->port.phy_mutex);
1000 }
1001
1002 /* calculates MF speed according to current linespeed and MF configuration */
1003 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1004 {
1005 u16 line_speed = bp->link_vars.line_speed;
1006 if (IS_MF(bp)) {
1007 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1008 bp->mf_config[BP_VN(bp)]);
1009
1010 /* Calculate the current MAX line speed limit for the MF
1011 * devices
1012 */
1013 if (IS_MF_SI(bp))
1014 line_speed = (line_speed * maxCfg) / 100;
1015 else { /* SD mode */
1016 u16 vn_max_rate = maxCfg * 100;
1017
1018 if (vn_max_rate < line_speed)
1019 line_speed = vn_max_rate;
1020 }
1021 }
1022
1023 return line_speed;
1024 }
1025
1026 /**
1027 * bnx2x_fill_report_data - fill link report data to report
1028 *
1029 * @bp: driver handle
1030 * @data: link state to update
1031 *
1032 * It uses a none-atomic bit operations because is called under the mutex.
1033 */
1034 static void bnx2x_fill_report_data(struct bnx2x *bp,
1035 struct bnx2x_link_report_data *data)
1036 {
1037 u16 line_speed = bnx2x_get_mf_speed(bp);
1038
1039 memset(data, 0, sizeof(*data));
1040
1041 /* Fill the report data: efective line speed */
1042 data->line_speed = line_speed;
1043
1044 /* Link is down */
1045 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1046 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1047 &data->link_report_flags);
1048
1049 /* Full DUPLEX */
1050 if (bp->link_vars.duplex == DUPLEX_FULL)
1051 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1052
1053 /* Rx Flow Control is ON */
1054 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1055 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1056
1057 /* Tx Flow Control is ON */
1058 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1059 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1060 }
1061
1062 /**
1063 * bnx2x_link_report - report link status to OS.
1064 *
1065 * @bp: driver handle
1066 *
1067 * Calls the __bnx2x_link_report() under the same locking scheme
1068 * as a link/PHY state managing code to ensure a consistent link
1069 * reporting.
1070 */
1071
1072 void bnx2x_link_report(struct bnx2x *bp)
1073 {
1074 bnx2x_acquire_phy_lock(bp);
1075 __bnx2x_link_report(bp);
1076 bnx2x_release_phy_lock(bp);
1077 }
1078
1079 /**
1080 * __bnx2x_link_report - report link status to OS.
1081 *
1082 * @bp: driver handle
1083 *
1084 * None atomic inmlementation.
1085 * Should be called under the phy_lock.
1086 */
1087 void __bnx2x_link_report(struct bnx2x *bp)
1088 {
1089 struct bnx2x_link_report_data cur_data;
1090
1091 /* reread mf_cfg */
1092 if (!CHIP_IS_E1(bp))
1093 bnx2x_read_mf_cfg(bp);
1094
1095 /* Read the current link report info */
1096 bnx2x_fill_report_data(bp, &cur_data);
1097
1098 /* Don't report link down or exactly the same link status twice */
1099 if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1100 (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1101 &bp->last_reported_link.link_report_flags) &&
1102 test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1103 &cur_data.link_report_flags)))
1104 return;
1105
1106 bp->link_cnt++;
1107
1108 /* We are going to report a new link parameters now -
1109 * remember the current data for the next time.
1110 */
1111 memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1112
1113 if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1114 &cur_data.link_report_flags)) {
1115 netif_carrier_off(bp->dev);
1116 netdev_err(bp->dev, "NIC Link is Down\n");
1117 return;
1118 } else {
1119 const char *duplex;
1120 const char *flow;
1121
1122 netif_carrier_on(bp->dev);
1123
1124 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1125 &cur_data.link_report_flags))
1126 duplex = "full";
1127 else
1128 duplex = "half";
1129
1130 /* Handle the FC at the end so that only these flags would be
1131 * possibly set. This way we may easily check if there is no FC
1132 * enabled.
1133 */
1134 if (cur_data.link_report_flags) {
1135 if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1136 &cur_data.link_report_flags)) {
1137 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1138 &cur_data.link_report_flags))
1139 flow = "ON - receive & transmit";
1140 else
1141 flow = "ON - receive";
1142 } else {
1143 flow = "ON - transmit";
1144 }
1145 } else {
1146 flow = "none";
1147 }
1148 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1149 cur_data.line_speed, duplex, flow);
1150 }
1151 }
1152
1153 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1154 {
1155 int i;
1156
1157 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1158 struct eth_rx_sge *sge;
1159
1160 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1161 sge->addr_hi =
1162 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1163 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1164
1165 sge->addr_lo =
1166 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1167 BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1168 }
1169 }
1170
1171 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1172 struct bnx2x_fastpath *fp, int last)
1173 {
1174 int i;
1175
1176 for (i = 0; i < last; i++) {
1177 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1178 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1179 u8 *data = first_buf->data;
1180
1181 if (data == NULL) {
1182 DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1183 continue;
1184 }
1185 if (tpa_info->tpa_state == BNX2X_TPA_START)
1186 dma_unmap_single(&bp->pdev->dev,
1187 dma_unmap_addr(first_buf, mapping),
1188 fp->rx_buf_size, DMA_FROM_DEVICE);
1189 bnx2x_frag_free(fp, data);
1190 first_buf->data = NULL;
1191 }
1192 }
1193
1194 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1195 {
1196 int j;
1197
1198 for_each_rx_queue_cnic(bp, j) {
1199 struct bnx2x_fastpath *fp = &bp->fp[j];
1200
1201 fp->rx_bd_cons = 0;
1202
1203 /* Activate BD ring */
1204 /* Warning!
1205 * this will generate an interrupt (to the TSTORM)
1206 * must only be done after chip is initialized
1207 */
1208 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1209 fp->rx_sge_prod);
1210 }
1211 }
1212
1213 void bnx2x_init_rx_rings(struct bnx2x *bp)
1214 {
1215 int func = BP_FUNC(bp);
1216 u16 ring_prod;
1217 int i, j;
1218
1219 /* Allocate TPA resources */
1220 for_each_eth_queue(bp, j) {
1221 struct bnx2x_fastpath *fp = &bp->fp[j];
1222
1223 DP(NETIF_MSG_IFUP,
1224 "mtu %d rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1225
1226 if (!fp->disable_tpa) {
1227 /* Fill the per-aggregtion pool */
1228 for (i = 0; i < MAX_AGG_QS(bp); i++) {
1229 struct bnx2x_agg_info *tpa_info =
1230 &fp->tpa_info[i];
1231 struct sw_rx_bd *first_buf =
1232 &tpa_info->first_buf;
1233
1234 first_buf->data = bnx2x_frag_alloc(fp);
1235 if (!first_buf->data) {
1236 BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1237 j);
1238 bnx2x_free_tpa_pool(bp, fp, i);
1239 fp->disable_tpa = 1;
1240 break;
1241 }
1242 dma_unmap_addr_set(first_buf, mapping, 0);
1243 tpa_info->tpa_state = BNX2X_TPA_STOP;
1244 }
1245
1246 /* "next page" elements initialization */
1247 bnx2x_set_next_page_sgl(fp);
1248
1249 /* set SGEs bit mask */
1250 bnx2x_init_sge_ring_bit_mask(fp);
1251
1252 /* Allocate SGEs and initialize the ring elements */
1253 for (i = 0, ring_prod = 0;
1254 i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1255
1256 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1257 BNX2X_ERR("was only able to allocate %d rx sges\n",
1258 i);
1259 BNX2X_ERR("disabling TPA for queue[%d]\n",
1260 j);
1261 /* Cleanup already allocated elements */
1262 bnx2x_free_rx_sge_range(bp, fp,
1263 ring_prod);
1264 bnx2x_free_tpa_pool(bp, fp,
1265 MAX_AGG_QS(bp));
1266 fp->disable_tpa = 1;
1267 ring_prod = 0;
1268 break;
1269 }
1270 ring_prod = NEXT_SGE_IDX(ring_prod);
1271 }
1272
1273 fp->rx_sge_prod = ring_prod;
1274 }
1275 }
1276
1277 for_each_eth_queue(bp, j) {
1278 struct bnx2x_fastpath *fp = &bp->fp[j];
1279
1280 fp->rx_bd_cons = 0;
1281
1282 /* Activate BD ring */
1283 /* Warning!
1284 * this will generate an interrupt (to the TSTORM)
1285 * must only be done after chip is initialized
1286 */
1287 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1288 fp->rx_sge_prod);
1289
1290 if (j != 0)
1291 continue;
1292
1293 if (CHIP_IS_E1(bp)) {
1294 REG_WR(bp, BAR_USTRORM_INTMEM +
1295 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1296 U64_LO(fp->rx_comp_mapping));
1297 REG_WR(bp, BAR_USTRORM_INTMEM +
1298 USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1299 U64_HI(fp->rx_comp_mapping));
1300 }
1301 }
1302 }
1303
1304 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1305 {
1306 u8 cos;
1307 struct bnx2x *bp = fp->bp;
1308
1309 for_each_cos_in_tx_queue(fp, cos) {
1310 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1311 unsigned pkts_compl = 0, bytes_compl = 0;
1312
1313 u16 sw_prod = txdata->tx_pkt_prod;
1314 u16 sw_cons = txdata->tx_pkt_cons;
1315
1316 while (sw_cons != sw_prod) {
1317 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1318 &pkts_compl, &bytes_compl);
1319 sw_cons++;
1320 }
1321
1322 netdev_tx_reset_queue(
1323 netdev_get_tx_queue(bp->dev,
1324 txdata->txq_index));
1325 }
1326 }
1327
1328 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1329 {
1330 int i;
1331
1332 for_each_tx_queue_cnic(bp, i) {
1333 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1334 }
1335 }
1336
1337 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1338 {
1339 int i;
1340
1341 for_each_eth_queue(bp, i) {
1342 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1343 }
1344 }
1345
1346 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1347 {
1348 struct bnx2x *bp = fp->bp;
1349 int i;
1350
1351 /* ring wasn't allocated */
1352 if (fp->rx_buf_ring == NULL)
1353 return;
1354
1355 for (i = 0; i < NUM_RX_BD; i++) {
1356 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1357 u8 *data = rx_buf->data;
1358
1359 if (data == NULL)
1360 continue;
1361 dma_unmap_single(&bp->pdev->dev,
1362 dma_unmap_addr(rx_buf, mapping),
1363 fp->rx_buf_size, DMA_FROM_DEVICE);
1364
1365 rx_buf->data = NULL;
1366 bnx2x_frag_free(fp, data);
1367 }
1368 }
1369
1370 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1371 {
1372 int j;
1373
1374 for_each_rx_queue_cnic(bp, j) {
1375 bnx2x_free_rx_bds(&bp->fp[j]);
1376 }
1377 }
1378
1379 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1380 {
1381 int j;
1382
1383 for_each_eth_queue(bp, j) {
1384 struct bnx2x_fastpath *fp = &bp->fp[j];
1385
1386 bnx2x_free_rx_bds(fp);
1387
1388 if (!fp->disable_tpa)
1389 bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1390 }
1391 }
1392
1393 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1394 {
1395 bnx2x_free_tx_skbs_cnic(bp);
1396 bnx2x_free_rx_skbs_cnic(bp);
1397 }
1398
1399 void bnx2x_free_skbs(struct bnx2x *bp)
1400 {
1401 bnx2x_free_tx_skbs(bp);
1402 bnx2x_free_rx_skbs(bp);
1403 }
1404
1405 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1406 {
1407 /* load old values */
1408 u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1409
1410 if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1411 /* leave all but MAX value */
1412 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1413
1414 /* set new MAX value */
1415 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1416 & FUNC_MF_CFG_MAX_BW_MASK;
1417
1418 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1419 }
1420 }
1421
1422 /**
1423 * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1424 *
1425 * @bp: driver handle
1426 * @nvecs: number of vectors to be released
1427 */
1428 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1429 {
1430 int i, offset = 0;
1431
1432 if (nvecs == offset)
1433 return;
1434 free_irq(bp->msix_table[offset].vector, bp->dev);
1435 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1436 bp->msix_table[offset].vector);
1437 offset++;
1438
1439 if (CNIC_SUPPORT(bp)) {
1440 if (nvecs == offset)
1441 return;
1442 offset++;
1443 }
1444
1445 for_each_eth_queue(bp, i) {
1446 if (nvecs == offset)
1447 return;
1448 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1449 i, bp->msix_table[offset].vector);
1450
1451 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1452 }
1453 }
1454
1455 void bnx2x_free_irq(struct bnx2x *bp)
1456 {
1457 if (bp->flags & USING_MSIX_FLAG &&
1458 !(bp->flags & USING_SINGLE_MSIX_FLAG))
1459 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1460 CNIC_SUPPORT(bp) + 1);
1461 else
1462 free_irq(bp->dev->irq, bp->dev);
1463 }
1464
1465 int bnx2x_enable_msix(struct bnx2x *bp)
1466 {
1467 int msix_vec = 0, i, rc, req_cnt;
1468
1469 bp->msix_table[msix_vec].entry = msix_vec;
1470 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1471 bp->msix_table[0].entry);
1472 msix_vec++;
1473
1474 /* Cnic requires an msix vector for itself */
1475 if (CNIC_SUPPORT(bp)) {
1476 bp->msix_table[msix_vec].entry = msix_vec;
1477 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1478 msix_vec, bp->msix_table[msix_vec].entry);
1479 msix_vec++;
1480 }
1481
1482 /* We need separate vectors for ETH queues only (not FCoE) */
1483 for_each_eth_queue(bp, i) {
1484 bp->msix_table[msix_vec].entry = msix_vec;
1485 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1486 msix_vec, msix_vec, i);
1487 msix_vec++;
1488 }
1489
1490 req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp) + 1;
1491
1492 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1493
1494 /*
1495 * reconfigure number of tx/rx queues according to available
1496 * MSI-X vectors
1497 */
1498 if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1499 /* how less vectors we will have? */
1500 int diff = req_cnt - rc;
1501
1502 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1503
1504 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1505
1506 if (rc) {
1507 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1508 goto no_msix;
1509 }
1510 /*
1511 * decrease number of queues by number of unallocated entries
1512 */
1513 bp->num_ethernet_queues -= diff;
1514 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1515
1516 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1517 bp->num_queues);
1518 } else if (rc > 0) {
1519 /* Get by with single vector */
1520 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1521 if (rc) {
1522 BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1523 rc);
1524 goto no_msix;
1525 }
1526
1527 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1528 bp->flags |= USING_SINGLE_MSIX_FLAG;
1529
1530 BNX2X_DEV_INFO("set number of queues to 1\n");
1531 bp->num_ethernet_queues = 1;
1532 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1533 } else if (rc < 0) {
1534 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1535 goto no_msix;
1536 }
1537
1538 bp->flags |= USING_MSIX_FLAG;
1539
1540 return 0;
1541
1542 no_msix:
1543 /* fall to INTx if not enough memory */
1544 if (rc == -ENOMEM)
1545 bp->flags |= DISABLE_MSI_FLAG;
1546
1547 return rc;
1548 }
1549
1550 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1551 {
1552 int i, rc, offset = 0;
1553
1554 rc = request_irq(bp->msix_table[offset++].vector,
1555 bnx2x_msix_sp_int, 0,
1556 bp->dev->name, bp->dev);
1557 if (rc) {
1558 BNX2X_ERR("request sp irq failed\n");
1559 return -EBUSY;
1560 }
1561
1562 if (CNIC_SUPPORT(bp))
1563 offset++;
1564
1565 for_each_eth_queue(bp, i) {
1566 struct bnx2x_fastpath *fp = &bp->fp[i];
1567 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1568 bp->dev->name, i);
1569
1570 rc = request_irq(bp->msix_table[offset].vector,
1571 bnx2x_msix_fp_int, 0, fp->name, fp);
1572 if (rc) {
1573 BNX2X_ERR("request fp #%d irq (%d) failed rc %d\n", i,
1574 bp->msix_table[offset].vector, rc);
1575 bnx2x_free_msix_irqs(bp, offset);
1576 return -EBUSY;
1577 }
1578
1579 offset++;
1580 }
1581
1582 i = BNX2X_NUM_ETH_QUEUES(bp);
1583 offset = 1 + CNIC_SUPPORT(bp);
1584 netdev_info(bp->dev, "using MSI-X IRQs: sp %d fp[%d] %d ... fp[%d] %d\n",
1585 bp->msix_table[0].vector,
1586 0, bp->msix_table[offset].vector,
1587 i - 1, bp->msix_table[offset + i - 1].vector);
1588
1589 return 0;
1590 }
1591
1592 int bnx2x_enable_msi(struct bnx2x *bp)
1593 {
1594 int rc;
1595
1596 rc = pci_enable_msi(bp->pdev);
1597 if (rc) {
1598 BNX2X_DEV_INFO("MSI is not attainable\n");
1599 return -1;
1600 }
1601 bp->flags |= USING_MSI_FLAG;
1602
1603 return 0;
1604 }
1605
1606 static int bnx2x_req_irq(struct bnx2x *bp)
1607 {
1608 unsigned long flags;
1609 unsigned int irq;
1610
1611 if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1612 flags = 0;
1613 else
1614 flags = IRQF_SHARED;
1615
1616 if (bp->flags & USING_MSIX_FLAG)
1617 irq = bp->msix_table[0].vector;
1618 else
1619 irq = bp->pdev->irq;
1620
1621 return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1622 }
1623
1624 static int bnx2x_setup_irqs(struct bnx2x *bp)
1625 {
1626 int rc = 0;
1627 if (bp->flags & USING_MSIX_FLAG &&
1628 !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1629 rc = bnx2x_req_msix_irqs(bp);
1630 if (rc)
1631 return rc;
1632 } else {
1633 bnx2x_ack_int(bp);
1634 rc = bnx2x_req_irq(bp);
1635 if (rc) {
1636 BNX2X_ERR("IRQ request failed rc %d, aborting\n", rc);
1637 return rc;
1638 }
1639 if (bp->flags & USING_MSI_FLAG) {
1640 bp->dev->irq = bp->pdev->irq;
1641 netdev_info(bp->dev, "using MSI IRQ %d\n",
1642 bp->dev->irq);
1643 }
1644 if (bp->flags & USING_MSIX_FLAG) {
1645 bp->dev->irq = bp->msix_table[0].vector;
1646 netdev_info(bp->dev, "using MSIX IRQ %d\n",
1647 bp->dev->irq);
1648 }
1649 }
1650
1651 return 0;
1652 }
1653
1654 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1655 {
1656 int i;
1657
1658 for_each_rx_queue_cnic(bp, i)
1659 napi_enable(&bnx2x_fp(bp, i, napi));
1660 }
1661
1662 static void bnx2x_napi_enable(struct bnx2x *bp)
1663 {
1664 int i;
1665
1666 for_each_eth_queue(bp, i)
1667 napi_enable(&bnx2x_fp(bp, i, napi));
1668 }
1669
1670 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1671 {
1672 int i;
1673
1674 for_each_rx_queue_cnic(bp, i)
1675 napi_disable(&bnx2x_fp(bp, i, napi));
1676 }
1677
1678 static void bnx2x_napi_disable(struct bnx2x *bp)
1679 {
1680 int i;
1681
1682 for_each_eth_queue(bp, i)
1683 napi_disable(&bnx2x_fp(bp, i, napi));
1684 }
1685
1686 void bnx2x_netif_start(struct bnx2x *bp)
1687 {
1688 if (netif_running(bp->dev)) {
1689 bnx2x_napi_enable(bp);
1690 if (CNIC_LOADED(bp))
1691 bnx2x_napi_enable_cnic(bp);
1692 bnx2x_int_enable(bp);
1693 if (bp->state == BNX2X_STATE_OPEN)
1694 netif_tx_wake_all_queues(bp->dev);
1695 }
1696 }
1697
1698 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1699 {
1700 bnx2x_int_disable_sync(bp, disable_hw);
1701 bnx2x_napi_disable(bp);
1702 if (CNIC_LOADED(bp))
1703 bnx2x_napi_disable_cnic(bp);
1704 }
1705
1706 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1707 {
1708 struct bnx2x *bp = netdev_priv(dev);
1709
1710 if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1711 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1712 u16 ether_type = ntohs(hdr->h_proto);
1713
1714 /* Skip VLAN tag if present */
1715 if (ether_type == ETH_P_8021Q) {
1716 struct vlan_ethhdr *vhdr =
1717 (struct vlan_ethhdr *)skb->data;
1718
1719 ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1720 }
1721
1722 /* If ethertype is FCoE or FIP - use FCoE ring */
1723 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1724 return bnx2x_fcoe_tx(bp, txq_index);
1725 }
1726
1727 /* select a non-FCoE queue */
1728 return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1729 }
1730
1731
1732 void bnx2x_set_num_queues(struct bnx2x *bp)
1733 {
1734 /* RSS queues */
1735 bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1736
1737 /* override in STORAGE SD modes */
1738 if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1739 bp->num_ethernet_queues = 1;
1740
1741 /* Add special queues */
1742 bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1743 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1744
1745 BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1746 }
1747
1748 /**
1749 * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1750 *
1751 * @bp: Driver handle
1752 *
1753 * We currently support for at most 16 Tx queues for each CoS thus we will
1754 * allocate a multiple of 16 for ETH L2 rings according to the value of the
1755 * bp->max_cos.
1756 *
1757 * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1758 * index after all ETH L2 indices.
1759 *
1760 * If the actual number of Tx queues (for each CoS) is less than 16 then there
1761 * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1762 * 16..31,...) with indicies that are not coupled with any real Tx queue.
1763 *
1764 * The proper configuration of skb->queue_mapping is handled by
1765 * bnx2x_select_queue() and __skb_tx_hash().
1766 *
1767 * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1768 * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1769 */
1770 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1771 {
1772 int rc, tx, rx;
1773
1774 tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1775 rx = BNX2X_NUM_ETH_QUEUES(bp);
1776
1777 /* account for fcoe queue */
1778 if (include_cnic && !NO_FCOE(bp)) {
1779 rx++;
1780 tx++;
1781 }
1782
1783 rc = netif_set_real_num_tx_queues(bp->dev, tx);
1784 if (rc) {
1785 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1786 return rc;
1787 }
1788 rc = netif_set_real_num_rx_queues(bp->dev, rx);
1789 if (rc) {
1790 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1791 return rc;
1792 }
1793
1794 DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1795 tx, rx);
1796
1797 return rc;
1798 }
1799
1800 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1801 {
1802 int i;
1803
1804 for_each_queue(bp, i) {
1805 struct bnx2x_fastpath *fp = &bp->fp[i];
1806 u32 mtu;
1807
1808 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1809 if (IS_FCOE_IDX(i))
1810 /*
1811 * Although there are no IP frames expected to arrive to
1812 * this ring we still want to add an
1813 * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1814 * overrun attack.
1815 */
1816 mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1817 else
1818 mtu = bp->dev->mtu;
1819 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1820 IP_HEADER_ALIGNMENT_PADDING +
1821 ETH_OVREHEAD +
1822 mtu +
1823 BNX2X_FW_RX_ALIGN_END;
1824 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1825 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1826 fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1827 else
1828 fp->rx_frag_size = 0;
1829 }
1830 }
1831
1832 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1833 {
1834 int i;
1835 u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1836
1837 /* Prepare the initial contents fo the indirection table if RSS is
1838 * enabled
1839 */
1840 for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1841 bp->rss_conf_obj.ind_table[i] =
1842 bp->fp->cl_id +
1843 ethtool_rxfh_indir_default(i, num_eth_queues);
1844
1845 /*
1846 * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1847 * per-port, so if explicit configuration is needed , do it only
1848 * for a PMF.
1849 *
1850 * For 57712 and newer on the other hand it's a per-function
1851 * configuration.
1852 */
1853 return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1854 }
1855
1856 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1857 bool config_hash)
1858 {
1859 struct bnx2x_config_rss_params params = {NULL};
1860
1861 /* Although RSS is meaningless when there is a single HW queue we
1862 * still need it enabled in order to have HW Rx hash generated.
1863 *
1864 * if (!is_eth_multi(bp))
1865 * bp->multi_mode = ETH_RSS_MODE_DISABLED;
1866 */
1867
1868 params.rss_obj = rss_obj;
1869
1870 __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1871
1872 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1873
1874 /* RSS configuration */
1875 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1876 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1877 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1878 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1879 if (rss_obj->udp_rss_v4)
1880 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1881 if (rss_obj->udp_rss_v6)
1882 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1883
1884 /* Hash bits */
1885 params.rss_result_mask = MULTI_MASK;
1886
1887 memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1888
1889 if (config_hash) {
1890 /* RSS keys */
1891 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1892 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1893 }
1894
1895 return bnx2x_config_rss(bp, &params);
1896 }
1897
1898 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1899 {
1900 struct bnx2x_func_state_params func_params = {NULL};
1901
1902 /* Prepare parameters for function state transitions */
1903 __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1904
1905 func_params.f_obj = &bp->func_obj;
1906 func_params.cmd = BNX2X_F_CMD_HW_INIT;
1907
1908 func_params.params.hw_init.load_phase = load_code;
1909
1910 return bnx2x_func_state_change(bp, &func_params);
1911 }
1912
1913 /*
1914 * Cleans the object that have internal lists without sending
1915 * ramrods. Should be run when interrutps are disabled.
1916 */
1917 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1918 {
1919 int rc;
1920 unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1921 struct bnx2x_mcast_ramrod_params rparam = {NULL};
1922 struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1923
1924 /***************** Cleanup MACs' object first *************************/
1925
1926 /* Wait for completion of requested */
1927 __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1928 /* Perform a dry cleanup */
1929 __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1930
1931 /* Clean ETH primary MAC */
1932 __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1933 rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1934 &ramrod_flags);
1935 if (rc != 0)
1936 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1937
1938 /* Cleanup UC list */
1939 vlan_mac_flags = 0;
1940 __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1941 rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1942 &ramrod_flags);
1943 if (rc != 0)
1944 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1945
1946 /***************** Now clean mcast object *****************************/
1947 rparam.mcast_obj = &bp->mcast_obj;
1948 __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1949
1950 /* Add a DEL command... */
1951 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1952 if (rc < 0)
1953 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1954 rc);
1955
1956 /* ...and wait until all pending commands are cleared */
1957 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1958 while (rc != 0) {
1959 if (rc < 0) {
1960 BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1961 rc);
1962 return;
1963 }
1964
1965 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1966 }
1967 }
1968
1969 #ifndef BNX2X_STOP_ON_ERROR
1970 #define LOAD_ERROR_EXIT(bp, label) \
1971 do { \
1972 (bp)->state = BNX2X_STATE_ERROR; \
1973 goto label; \
1974 } while (0)
1975
1976 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1977 do { \
1978 bp->cnic_loaded = false; \
1979 goto label; \
1980 } while (0)
1981 #else /*BNX2X_STOP_ON_ERROR*/
1982 #define LOAD_ERROR_EXIT(bp, label) \
1983 do { \
1984 (bp)->state = BNX2X_STATE_ERROR; \
1985 (bp)->panic = 1; \
1986 return -EBUSY; \
1987 } while (0)
1988 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1989 do { \
1990 bp->cnic_loaded = false; \
1991 (bp)->panic = 1; \
1992 return -EBUSY; \
1993 } while (0)
1994 #endif /*BNX2X_STOP_ON_ERROR*/
1995
1996 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1997 {
1998 /* build FW version dword */
1999 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2000 (BCM_5710_FW_MINOR_VERSION << 8) +
2001 (BCM_5710_FW_REVISION_VERSION << 16) +
2002 (BCM_5710_FW_ENGINEERING_VERSION << 24);
2003
2004 /* read loaded FW from chip */
2005 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2006
2007 DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
2008
2009 if (loaded_fw != my_fw) {
2010 if (is_err)
2011 BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
2012 loaded_fw, my_fw);
2013 return false;
2014 }
2015
2016 return true;
2017 }
2018
2019 /**
2020 * bnx2x_bz_fp - zero content of the fastpath structure.
2021 *
2022 * @bp: driver handle
2023 * @index: fastpath index to be zeroed
2024 *
2025 * Makes sure the contents of the bp->fp[index].napi is kept
2026 * intact.
2027 */
2028 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2029 {
2030 struct bnx2x_fastpath *fp = &bp->fp[index];
2031 struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
2032
2033 int cos;
2034 struct napi_struct orig_napi = fp->napi;
2035 struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2036 /* bzero bnx2x_fastpath contents */
2037 if (bp->stats_init) {
2038 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2039 memset(fp, 0, sizeof(*fp));
2040 } else {
2041 /* Keep Queue statistics */
2042 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
2043 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
2044
2045 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
2046 GFP_KERNEL);
2047 if (tmp_eth_q_stats)
2048 memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
2049 sizeof(struct bnx2x_eth_q_stats));
2050
2051 tmp_eth_q_stats_old =
2052 kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
2053 GFP_KERNEL);
2054 if (tmp_eth_q_stats_old)
2055 memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
2056 sizeof(struct bnx2x_eth_q_stats_old));
2057
2058 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2059 memset(fp, 0, sizeof(*fp));
2060
2061 if (tmp_eth_q_stats) {
2062 memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
2063 sizeof(struct bnx2x_eth_q_stats));
2064 kfree(tmp_eth_q_stats);
2065 }
2066
2067 if (tmp_eth_q_stats_old) {
2068 memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
2069 sizeof(struct bnx2x_eth_q_stats_old));
2070 kfree(tmp_eth_q_stats_old);
2071 }
2072
2073 }
2074
2075 /* Restore the NAPI object as it has been already initialized */
2076 fp->napi = orig_napi;
2077 fp->tpa_info = orig_tpa_info;
2078 fp->bp = bp;
2079 fp->index = index;
2080 if (IS_ETH_FP(fp))
2081 fp->max_cos = bp->max_cos;
2082 else
2083 /* Special queues support only one CoS */
2084 fp->max_cos = 1;
2085
2086 /* Init txdata pointers */
2087 if (IS_FCOE_FP(fp))
2088 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2089 if (IS_ETH_FP(fp))
2090 for_each_cos_in_tx_queue(fp, cos)
2091 fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2092 BNX2X_NUM_ETH_QUEUES(bp) + index];
2093
2094 /*
2095 * set the tpa flag for each queue. The tpa flag determines the queue
2096 * minimal size so it must be set prior to queue memory allocation
2097 */
2098 fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2099 (bp->flags & GRO_ENABLE_FLAG &&
2100 bnx2x_mtu_allows_gro(bp->dev->mtu)));
2101 if (bp->flags & TPA_ENABLE_FLAG)
2102 fp->mode = TPA_MODE_LRO;
2103 else if (bp->flags & GRO_ENABLE_FLAG)
2104 fp->mode = TPA_MODE_GRO;
2105
2106 /* We don't want TPA on an FCoE L2 ring */
2107 if (IS_FCOE_FP(fp))
2108 fp->disable_tpa = 1;
2109 }
2110
2111 int bnx2x_load_cnic(struct bnx2x *bp)
2112 {
2113 int i, rc, port = BP_PORT(bp);
2114
2115 DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2116
2117 mutex_init(&bp->cnic_mutex);
2118
2119 rc = bnx2x_alloc_mem_cnic(bp);
2120 if (rc) {
2121 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2122 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2123 }
2124
2125 rc = bnx2x_alloc_fp_mem_cnic(bp);
2126 if (rc) {
2127 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2128 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2129 }
2130
2131 /* Update the number of queues with the cnic queues */
2132 rc = bnx2x_set_real_num_queues(bp, 1);
2133 if (rc) {
2134 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2135 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2136 }
2137
2138 /* Add all CNIC NAPI objects */
2139 bnx2x_add_all_napi_cnic(bp);
2140 DP(NETIF_MSG_IFUP, "cnic napi added\n");
2141 bnx2x_napi_enable_cnic(bp);
2142
2143 rc = bnx2x_init_hw_func_cnic(bp);
2144 if (rc)
2145 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2146
2147 bnx2x_nic_init_cnic(bp);
2148
2149 /* Enable Timer scan */
2150 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2151
2152 for_each_cnic_queue(bp, i) {
2153 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2154 if (rc) {
2155 BNX2X_ERR("Queue setup failed\n");
2156 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2157 }
2158 }
2159
2160 /* Initialize Rx filter. */
2161 netif_addr_lock_bh(bp->dev);
2162 bnx2x_set_rx_mode(bp->dev);
2163 netif_addr_unlock_bh(bp->dev);
2164
2165 /* re-read iscsi info */
2166 bnx2x_get_iscsi_info(bp);
2167 bnx2x_setup_cnic_irq_info(bp);
2168 bnx2x_setup_cnic_info(bp);
2169 bp->cnic_loaded = true;
2170 if (bp->state == BNX2X_STATE_OPEN)
2171 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2172
2173
2174 DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2175
2176 return 0;
2177
2178 #ifndef BNX2X_STOP_ON_ERROR
2179 load_error_cnic2:
2180 /* Disable Timer scan */
2181 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2182
2183 load_error_cnic1:
2184 bnx2x_napi_disable_cnic(bp);
2185 /* Update the number of queues without the cnic queues */
2186 rc = bnx2x_set_real_num_queues(bp, 0);
2187 if (rc)
2188 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2189 load_error_cnic0:
2190 BNX2X_ERR("CNIC-related load failed\n");
2191 bnx2x_free_fp_mem_cnic(bp);
2192 bnx2x_free_mem_cnic(bp);
2193 return rc;
2194 #endif /* ! BNX2X_STOP_ON_ERROR */
2195 }
2196
2197
2198 /* must be called with rtnl_lock */
2199 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2200 {
2201 int port = BP_PORT(bp);
2202 u32 load_code;
2203 int i, rc;
2204
2205 DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2206 DP(NETIF_MSG_IFUP,
2207 "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2208
2209 #ifdef BNX2X_STOP_ON_ERROR
2210 if (unlikely(bp->panic)) {
2211 BNX2X_ERR("Can't load NIC when there is panic\n");
2212 return -EPERM;
2213 }
2214 #endif
2215
2216 bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2217
2218 /* Set the initial link reported state to link down */
2219 bnx2x_acquire_phy_lock(bp);
2220 memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2221 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2222 &bp->last_reported_link.link_report_flags);
2223 bnx2x_release_phy_lock(bp);
2224
2225 /* must be called before memory allocation and HW init */
2226 bnx2x_ilt_set_info(bp);
2227
2228 /*
2229 * Zero fastpath structures preserving invariants like napi, which are
2230 * allocated only once, fp index, max_cos, bp pointer.
2231 * Also set fp->disable_tpa and txdata_ptr.
2232 */
2233 DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2234 for_each_queue(bp, i)
2235 bnx2x_bz_fp(bp, i);
2236 memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2237 bp->num_cnic_queues) *
2238 sizeof(struct bnx2x_fp_txdata));
2239
2240 bp->fcoe_init = false;
2241
2242 /* Set the receive queues buffer size */
2243 bnx2x_set_rx_buf_size(bp);
2244
2245 if (bnx2x_alloc_mem(bp))
2246 return -ENOMEM;
2247
2248 /* As long as bnx2x_alloc_mem() may possibly update
2249 * bp->num_queues, bnx2x_set_real_num_queues() should always
2250 * come after it. At this stage cnic queues are not counted.
2251 */
2252 rc = bnx2x_set_real_num_queues(bp, 0);
2253 if (rc) {
2254 BNX2X_ERR("Unable to set real_num_queues\n");
2255 LOAD_ERROR_EXIT(bp, load_error0);
2256 }
2257
2258 /* configure multi cos mappings in kernel.
2259 * this configuration may be overriden by a multi class queue discipline
2260 * or by a dcbx negotiation result.
2261 */
2262 bnx2x_setup_tc(bp->dev, bp->max_cos);
2263
2264 /* Add all NAPI objects */
2265 bnx2x_add_all_napi(bp);
2266 DP(NETIF_MSG_IFUP, "napi added\n");
2267 bnx2x_napi_enable(bp);
2268
2269 /* set pf load just before approaching the MCP */
2270 bnx2x_set_pf_load(bp);
2271
2272 /* Send LOAD_REQUEST command to MCP
2273 * Returns the type of LOAD command:
2274 * if it is the first port to be initialized
2275 * common blocks should be initialized, otherwise - not
2276 */
2277 if (!BP_NOMCP(bp)) {
2278 /* init fw_seq */
2279 bp->fw_seq =
2280 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2281 DRV_MSG_SEQ_NUMBER_MASK);
2282 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2283
2284 /* Get current FW pulse sequence */
2285 bp->fw_drv_pulse_wr_seq =
2286 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2287 DRV_PULSE_SEQ_MASK);
2288 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2289
2290 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2291 DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2292 if (!load_code) {
2293 BNX2X_ERR("MCP response failure, aborting\n");
2294 rc = -EBUSY;
2295 LOAD_ERROR_EXIT(bp, load_error1);
2296 }
2297 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2298 BNX2X_ERR("Driver load refused\n");
2299 rc = -EBUSY; /* other port in diagnostic mode */
2300 LOAD_ERROR_EXIT(bp, load_error1);
2301 }
2302 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2303 load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2304 /* abort nic load if version mismatch */
2305 if (!bnx2x_test_firmware_version(bp, true)) {
2306 rc = -EBUSY;
2307 LOAD_ERROR_EXIT(bp, load_error2);
2308 }
2309 }
2310
2311 } else {
2312 int path = BP_PATH(bp);
2313
2314 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d] %d, %d, %d\n",
2315 path, load_count[path][0], load_count[path][1],
2316 load_count[path][2]);
2317 load_count[path][0]++;
2318 load_count[path][1 + port]++;
2319 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d] %d, %d, %d\n",
2320 path, load_count[path][0], load_count[path][1],
2321 load_count[path][2]);
2322 if (load_count[path][0] == 1)
2323 load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2324 else if (load_count[path][1 + port] == 1)
2325 load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2326 else
2327 load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2328 }
2329
2330 if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2331 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2332 (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2333 bp->port.pmf = 1;
2334 /*
2335 * We need the barrier to ensure the ordering between the
2336 * writing to bp->port.pmf here and reading it from the
2337 * bnx2x_periodic_task().
2338 */
2339 smp_mb();
2340 } else
2341 bp->port.pmf = 0;
2342
2343 DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2344
2345 /* Init Function state controlling object */
2346 bnx2x__init_func_obj(bp);
2347
2348 /* Initialize HW */
2349 rc = bnx2x_init_hw(bp, load_code);
2350 if (rc) {
2351 BNX2X_ERR("HW init failed, aborting\n");
2352 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2353 LOAD_ERROR_EXIT(bp, load_error2);
2354 }
2355
2356 /* Connect to IRQs */
2357 rc = bnx2x_setup_irqs(bp);
2358 if (rc) {
2359 BNX2X_ERR("IRQs setup failed\n");
2360 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2361 LOAD_ERROR_EXIT(bp, load_error2);
2362 }
2363
2364 /* Setup NIC internals and enable interrupts */
2365 bnx2x_nic_init(bp, load_code);
2366
2367 /* Init per-function objects */
2368 bnx2x_init_bp_objs(bp);
2369
2370 if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2371 (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2372 (bp->common.shmem2_base)) {
2373 if (SHMEM2_HAS(bp, dcc_support))
2374 SHMEM2_WR(bp, dcc_support,
2375 (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2376 SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2377 if (SHMEM2_HAS(bp, afex_driver_support))
2378 SHMEM2_WR(bp, afex_driver_support,
2379 SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2380 }
2381
2382 /* Set AFEX default VLAN tag to an invalid value */
2383 bp->afex_def_vlan_tag = -1;
2384
2385 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2386 rc = bnx2x_func_start(bp);
2387 if (rc) {
2388 BNX2X_ERR("Function start failed!\n");
2389 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2390 LOAD_ERROR_EXIT(bp, load_error3);
2391 }
2392
2393 /* Send LOAD_DONE command to MCP */
2394 if (!BP_NOMCP(bp)) {
2395 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2396 if (!load_code) {
2397 BNX2X_ERR("MCP response failure, aborting\n");
2398 rc = -EBUSY;
2399 LOAD_ERROR_EXIT(bp, load_error3);
2400 }
2401 }
2402
2403 rc = bnx2x_setup_leading(bp);
2404 if (rc) {
2405 BNX2X_ERR("Setup leading failed!\n");
2406 LOAD_ERROR_EXIT(bp, load_error3);
2407 }
2408
2409 for_each_nondefault_eth_queue(bp, i) {
2410 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2411 if (rc) {
2412 BNX2X_ERR("Queue setup failed\n");
2413 LOAD_ERROR_EXIT(bp, load_error3);
2414 }
2415 }
2416
2417 rc = bnx2x_init_rss_pf(bp);
2418 if (rc) {
2419 BNX2X_ERR("PF RSS init failed\n");
2420 LOAD_ERROR_EXIT(bp, load_error3);
2421 }
2422
2423 /* Now when Clients are configured we are ready to work */
2424 bp->state = BNX2X_STATE_OPEN;
2425
2426 /* Configure a ucast MAC */
2427 rc = bnx2x_set_eth_mac(bp, true);
2428 if (rc) {
2429 BNX2X_ERR("Setting Ethernet MAC failed\n");
2430 LOAD_ERROR_EXIT(bp, load_error3);
2431 }
2432
2433 if (bp->pending_max) {
2434 bnx2x_update_max_mf_config(bp, bp->pending_max);
2435 bp->pending_max = 0;
2436 }
2437
2438 if (bp->port.pmf)
2439 bnx2x_initial_phy_init(bp, load_mode);
2440 bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2441
2442 /* Start fast path */
2443
2444 /* Initialize Rx filter. */
2445 netif_addr_lock_bh(bp->dev);
2446 bnx2x_set_rx_mode(bp->dev);
2447 netif_addr_unlock_bh(bp->dev);
2448
2449 /* Start the Tx */
2450 switch (load_mode) {
2451 case LOAD_NORMAL:
2452 /* Tx queue should be only reenabled */
2453 netif_tx_wake_all_queues(bp->dev);
2454 break;
2455
2456 case LOAD_OPEN:
2457 netif_tx_start_all_queues(bp->dev);
2458 smp_mb__after_clear_bit();
2459 break;
2460
2461 case LOAD_DIAG:
2462 case LOAD_LOOPBACK_EXT:
2463 bp->state = BNX2X_STATE_DIAG;
2464 break;
2465
2466 default:
2467 break;
2468 }
2469
2470 if (bp->port.pmf)
2471 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2472 else
2473 bnx2x__link_status_update(bp);
2474
2475 /* start the timer */
2476 mod_timer(&bp->timer, jiffies + bp->current_interval);
2477
2478 if (CNIC_ENABLED(bp))
2479 bnx2x_load_cnic(bp);
2480
2481 /* mark driver is loaded in shmem2 */
2482 if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2483 u32 val;
2484 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2485 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2486 val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2487 DRV_FLAGS_CAPABILITIES_LOADED_L2);
2488 }
2489
2490 /* Wait for all pending SP commands to complete */
2491 if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2492 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2493 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2494 return -EBUSY;
2495 }
2496
2497 /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2498 if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2499 bnx2x_dcbx_init(bp, false);
2500
2501 DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2502
2503 return 0;
2504
2505 #ifndef BNX2X_STOP_ON_ERROR
2506 load_error3:
2507 bnx2x_int_disable_sync(bp, 1);
2508
2509 /* Clean queueable objects */
2510 bnx2x_squeeze_objects(bp);
2511
2512 /* Free SKBs, SGEs, TPA pool and driver internals */
2513 bnx2x_free_skbs(bp);
2514 for_each_rx_queue(bp, i)
2515 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2516
2517 /* Release IRQs */
2518 bnx2x_free_irq(bp);
2519 load_error2:
2520 if (!BP_NOMCP(bp)) {
2521 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2522 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2523 }
2524
2525 bp->port.pmf = 0;
2526 load_error1:
2527 bnx2x_napi_disable(bp);
2528 /* clear pf_load status, as it was already set */
2529 bnx2x_clear_pf_load(bp);
2530 load_error0:
2531 bnx2x_free_mem(bp);
2532
2533 return rc;
2534 #endif /* ! BNX2X_STOP_ON_ERROR */
2535 }
2536
2537 /* must be called with rtnl_lock */
2538 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2539 {
2540 int i;
2541 bool global = false;
2542
2543 DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2544
2545 /* mark driver is unloaded in shmem2 */
2546 if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2547 u32 val;
2548 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2549 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2550 val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2551 }
2552
2553 if ((bp->state == BNX2X_STATE_CLOSED) ||
2554 (bp->state == BNX2X_STATE_ERROR)) {
2555 /* We can get here if the driver has been unloaded
2556 * during parity error recovery and is either waiting for a
2557 * leader to complete or for other functions to unload and
2558 * then ifdown has been issued. In this case we want to
2559 * unload and let other functions to complete a recovery
2560 * process.
2561 */
2562 bp->recovery_state = BNX2X_RECOVERY_DONE;
2563 bp->is_leader = 0;
2564 bnx2x_release_leader_lock(bp);
2565 smp_mb();
2566
2567 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2568 BNX2X_ERR("Can't unload in closed or error state\n");
2569 return -EINVAL;
2570 }
2571
2572 /*
2573 * It's important to set the bp->state to the value different from
2574 * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2575 * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2576 */
2577 bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2578 smp_mb();
2579
2580 if (CNIC_LOADED(bp))
2581 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2582
2583 /* Stop Tx */
2584 bnx2x_tx_disable(bp);
2585 netdev_reset_tc(bp->dev);
2586
2587 bp->rx_mode = BNX2X_RX_MODE_NONE;
2588
2589 del_timer_sync(&bp->timer);
2590
2591 /* Set ALWAYS_ALIVE bit in shmem */
2592 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2593
2594 bnx2x_drv_pulse(bp);
2595
2596 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2597 bnx2x_save_statistics(bp);
2598
2599 /* Cleanup the chip if needed */
2600 if (unload_mode != UNLOAD_RECOVERY)
2601 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2602 else {
2603 /* Send the UNLOAD_REQUEST to the MCP */
2604 bnx2x_send_unload_req(bp, unload_mode);
2605
2606 /*
2607 * Prevent transactions to host from the functions on the
2608 * engine that doesn't reset global blocks in case of global
2609 * attention once gloabl blocks are reset and gates are opened
2610 * (the engine which leader will perform the recovery
2611 * last).
2612 */
2613 if (!CHIP_IS_E1x(bp))
2614 bnx2x_pf_disable(bp);
2615
2616 /* Disable HW interrupts, NAPI */
2617 bnx2x_netif_stop(bp, 1);
2618 /* Delete all NAPI objects */
2619 bnx2x_del_all_napi(bp);
2620 if (CNIC_LOADED(bp))
2621 bnx2x_del_all_napi_cnic(bp);
2622 /* Release IRQs */
2623 bnx2x_free_irq(bp);
2624
2625 /* Report UNLOAD_DONE to MCP */
2626 bnx2x_send_unload_done(bp, false);
2627 }
2628
2629 /*
2630 * At this stage no more interrupts will arrive so we may safly clean
2631 * the queueable objects here in case they failed to get cleaned so far.
2632 */
2633 bnx2x_squeeze_objects(bp);
2634
2635 /* There should be no more pending SP commands at this stage */
2636 bp->sp_state = 0;
2637
2638 bp->port.pmf = 0;
2639
2640 /* Free SKBs, SGEs, TPA pool and driver internals */
2641 bnx2x_free_skbs(bp);
2642 if (CNIC_LOADED(bp))
2643 bnx2x_free_skbs_cnic(bp);
2644 for_each_rx_queue(bp, i)
2645 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2646
2647 if (CNIC_LOADED(bp)) {
2648 bnx2x_free_fp_mem_cnic(bp);
2649 bnx2x_free_mem_cnic(bp);
2650 }
2651 bnx2x_free_mem(bp);
2652
2653 bp->state = BNX2X_STATE_CLOSED;
2654 bp->cnic_loaded = false;
2655
2656 /* Check if there are pending parity attentions. If there are - set
2657 * RECOVERY_IN_PROGRESS.
2658 */
2659 if (bnx2x_chk_parity_attn(bp, &global, false)) {
2660 bnx2x_set_reset_in_progress(bp);
2661
2662 /* Set RESET_IS_GLOBAL if needed */
2663 if (global)
2664 bnx2x_set_reset_global(bp);
2665 }
2666
2667
2668 /* The last driver must disable a "close the gate" if there is no
2669 * parity attention or "process kill" pending.
2670 */
2671 if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2672 bnx2x_disable_close_the_gate(bp);
2673
2674 DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2675
2676 return 0;
2677 }
2678
2679 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2680 {
2681 u16 pmcsr;
2682
2683 /* If there is no power capability, silently succeed */
2684 if (!bp->pm_cap) {
2685 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2686 return 0;
2687 }
2688
2689 pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2690
2691 switch (state) {
2692 case PCI_D0:
2693 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2694 ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2695 PCI_PM_CTRL_PME_STATUS));
2696
2697 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2698 /* delay required during transition out of D3hot */
2699 msleep(20);
2700 break;
2701
2702 case PCI_D3hot:
2703 /* If there are other clients above don't
2704 shut down the power */
2705 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2706 return 0;
2707 /* Don't shut down the power for emulation and FPGA */
2708 if (CHIP_REV_IS_SLOW(bp))
2709 return 0;
2710
2711 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2712 pmcsr |= 3;
2713
2714 if (bp->wol)
2715 pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2716
2717 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2718 pmcsr);
2719
2720 /* No more memory access after this point until
2721 * device is brought back to D0.
2722 */
2723 break;
2724
2725 default:
2726 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2727 return -EINVAL;
2728 }
2729 return 0;
2730 }
2731
2732 /*
2733 * net_device service functions
2734 */
2735 int bnx2x_poll(struct napi_struct *napi, int budget)
2736 {
2737 int work_done = 0;
2738 u8 cos;
2739 struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2740 napi);
2741 struct bnx2x *bp = fp->bp;
2742
2743 while (1) {
2744 #ifdef BNX2X_STOP_ON_ERROR
2745 if (unlikely(bp->panic)) {
2746 napi_complete(napi);
2747 return 0;
2748 }
2749 #endif
2750
2751 for_each_cos_in_tx_queue(fp, cos)
2752 if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2753 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2754
2755
2756 if (bnx2x_has_rx_work(fp)) {
2757 work_done += bnx2x_rx_int(fp, budget - work_done);
2758
2759 /* must not complete if we consumed full budget */
2760 if (work_done >= budget)
2761 break;
2762 }
2763
2764 /* Fall out from the NAPI loop if needed */
2765 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2766
2767 /* No need to update SB for FCoE L2 ring as long as
2768 * it's connected to the default SB and the SB
2769 * has been updated when NAPI was scheduled.
2770 */
2771 if (IS_FCOE_FP(fp)) {
2772 napi_complete(napi);
2773 break;
2774 }
2775 bnx2x_update_fpsb_idx(fp);
2776 /* bnx2x_has_rx_work() reads the status block,
2777 * thus we need to ensure that status block indices
2778 * have been actually read (bnx2x_update_fpsb_idx)
2779 * prior to this check (bnx2x_has_rx_work) so that
2780 * we won't write the "newer" value of the status block
2781 * to IGU (if there was a DMA right after
2782 * bnx2x_has_rx_work and if there is no rmb, the memory
2783 * reading (bnx2x_update_fpsb_idx) may be postponed
2784 * to right before bnx2x_ack_sb). In this case there
2785 * will never be another interrupt until there is
2786 * another update of the status block, while there
2787 * is still unhandled work.
2788 */
2789 rmb();
2790
2791 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2792 napi_complete(napi);
2793 /* Re-enable interrupts */
2794 DP(NETIF_MSG_RX_STATUS,
2795 "Update index to %d\n", fp->fp_hc_idx);
2796 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2797 le16_to_cpu(fp->fp_hc_idx),
2798 IGU_INT_ENABLE, 1);
2799 break;
2800 }
2801 }
2802 }
2803
2804 return work_done;
2805 }
2806
2807 /* we split the first BD into headers and data BDs
2808 * to ease the pain of our fellow microcode engineers
2809 * we use one mapping for both BDs
2810 */
2811 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2812 struct bnx2x_fp_txdata *txdata,
2813 struct sw_tx_bd *tx_buf,
2814 struct eth_tx_start_bd **tx_bd, u16 hlen,
2815 u16 bd_prod, int nbd)
2816 {
2817 struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2818 struct eth_tx_bd *d_tx_bd;
2819 dma_addr_t mapping;
2820 int old_len = le16_to_cpu(h_tx_bd->nbytes);
2821
2822 /* first fix first BD */
2823 h_tx_bd->nbd = cpu_to_le16(nbd);
2824 h_tx_bd->nbytes = cpu_to_le16(hlen);
2825
2826 DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2827 h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2828
2829 /* now get a new data BD
2830 * (after the pbd) and fill it */
2831 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2832 d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2833
2834 mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2835 le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2836
2837 d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2838 d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2839 d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2840
2841 /* this marks the BD as one that has no individual mapping */
2842 tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2843
2844 DP(NETIF_MSG_TX_QUEUED,
2845 "TSO split data size is %d (%x:%x)\n",
2846 d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2847
2848 /* update tx_bd */
2849 *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2850
2851 return bd_prod;
2852 }
2853
2854 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2855 {
2856 if (fix > 0)
2857 csum = (u16) ~csum_fold(csum_sub(csum,
2858 csum_partial(t_header - fix, fix, 0)));
2859
2860 else if (fix < 0)
2861 csum = (u16) ~csum_fold(csum_add(csum,
2862 csum_partial(t_header, -fix, 0)));
2863
2864 return swab16(csum);
2865 }
2866
2867 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2868 {
2869 u32 rc;
2870
2871 if (skb->ip_summed != CHECKSUM_PARTIAL)
2872 rc = XMIT_PLAIN;
2873
2874 else {
2875 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2876 rc = XMIT_CSUM_V6;
2877 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2878 rc |= XMIT_CSUM_TCP;
2879
2880 } else {
2881 rc = XMIT_CSUM_V4;
2882 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2883 rc |= XMIT_CSUM_TCP;
2884 }
2885 }
2886
2887 if (skb_is_gso_v6(skb))
2888 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2889 else if (skb_is_gso(skb))
2890 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2891
2892 return rc;
2893 }
2894
2895 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2896 /* check if packet requires linearization (packet is too fragmented)
2897 no need to check fragmentation if page size > 8K (there will be no
2898 violation to FW restrictions) */
2899 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2900 u32 xmit_type)
2901 {
2902 int to_copy = 0;
2903 int hlen = 0;
2904 int first_bd_sz = 0;
2905
2906 /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2907 if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2908
2909 if (xmit_type & XMIT_GSO) {
2910 unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2911 /* Check if LSO packet needs to be copied:
2912 3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2913 int wnd_size = MAX_FETCH_BD - 3;
2914 /* Number of windows to check */
2915 int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2916 int wnd_idx = 0;
2917 int frag_idx = 0;
2918 u32 wnd_sum = 0;
2919
2920 /* Headers length */
2921 hlen = (int)(skb_transport_header(skb) - skb->data) +
2922 tcp_hdrlen(skb);
2923
2924 /* Amount of data (w/o headers) on linear part of SKB*/
2925 first_bd_sz = skb_headlen(skb) - hlen;
2926
2927 wnd_sum = first_bd_sz;
2928
2929 /* Calculate the first sum - it's special */
2930 for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2931 wnd_sum +=
2932 skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2933
2934 /* If there was data on linear skb data - check it */
2935 if (first_bd_sz > 0) {
2936 if (unlikely(wnd_sum < lso_mss)) {
2937 to_copy = 1;
2938 goto exit_lbl;
2939 }
2940
2941 wnd_sum -= first_bd_sz;
2942 }
2943
2944 /* Others are easier: run through the frag list and
2945 check all windows */
2946 for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2947 wnd_sum +=
2948 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2949
2950 if (unlikely(wnd_sum < lso_mss)) {
2951 to_copy = 1;
2952 break;
2953 }
2954 wnd_sum -=
2955 skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2956 }
2957 } else {
2958 /* in non-LSO too fragmented packet should always
2959 be linearized */
2960 to_copy = 1;
2961 }
2962 }
2963
2964 exit_lbl:
2965 if (unlikely(to_copy))
2966 DP(NETIF_MSG_TX_QUEUED,
2967 "Linearization IS REQUIRED for %s packet. num_frags %d hlen %d first_bd_sz %d\n",
2968 (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2969 skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2970
2971 return to_copy;
2972 }
2973 #endif
2974
2975 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2976 u32 xmit_type)
2977 {
2978 *parsing_data |= (skb_shinfo(skb)->gso_size <<
2979 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2980 ETH_TX_PARSE_BD_E2_LSO_MSS;
2981 if ((xmit_type & XMIT_GSO_V6) &&
2982 (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2983 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2984 }
2985
2986 /**
2987 * bnx2x_set_pbd_gso - update PBD in GSO case.
2988 *
2989 * @skb: packet skb
2990 * @pbd: parse BD
2991 * @xmit_type: xmit flags
2992 */
2993 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2994 struct eth_tx_parse_bd_e1x *pbd,
2995 u32 xmit_type)
2996 {
2997 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2998 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2999 pbd->tcp_flags = pbd_tcp_flags(skb);
3000
3001 if (xmit_type & XMIT_GSO_V4) {
3002 pbd->ip_id = swab16(ip_hdr(skb)->id);
3003 pbd->tcp_pseudo_csum =
3004 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3005 ip_hdr(skb)->daddr,
3006 0, IPPROTO_TCP, 0));
3007
3008 } else
3009 pbd->tcp_pseudo_csum =
3010 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3011 &ipv6_hdr(skb)->daddr,
3012 0, IPPROTO_TCP, 0));
3013
3014 pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
3015 }
3016
3017 /**
3018 * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3019 *
3020 * @bp: driver handle
3021 * @skb: packet skb
3022 * @parsing_data: data to be updated
3023 * @xmit_type: xmit flags
3024 *
3025 * 57712 related
3026 */
3027 static inline u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3028 u32 *parsing_data, u32 xmit_type)
3029 {
3030 *parsing_data |=
3031 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3032 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
3033 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
3034
3035 if (xmit_type & XMIT_CSUM_TCP) {
3036 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3037 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3038 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3039
3040 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3041 } else
3042 /* We support checksum offload for TCP and UDP only.
3043 * No need to pass the UDP header length - it's a constant.
3044 */
3045 return skb_transport_header(skb) +
3046 sizeof(struct udphdr) - skb->data;
3047 }
3048
3049 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3050 struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
3051 {
3052 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3053
3054 if (xmit_type & XMIT_CSUM_V4)
3055 tx_start_bd->bd_flags.as_bitfield |=
3056 ETH_TX_BD_FLAGS_IP_CSUM;
3057 else
3058 tx_start_bd->bd_flags.as_bitfield |=
3059 ETH_TX_BD_FLAGS_IPV6;
3060
3061 if (!(xmit_type & XMIT_CSUM_TCP))
3062 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3063 }
3064
3065 /**
3066 * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3067 *
3068 * @bp: driver handle
3069 * @skb: packet skb
3070 * @pbd: parse BD to be updated
3071 * @xmit_type: xmit flags
3072 */
3073 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3074 struct eth_tx_parse_bd_e1x *pbd,
3075 u32 xmit_type)
3076 {
3077 u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3078
3079 /* for now NS flag is not used in Linux */
3080 pbd->global_data =
3081 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3082 ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3083
3084 pbd->ip_hlen_w = (skb_transport_header(skb) -
3085 skb_network_header(skb)) >> 1;
3086
3087 hlen += pbd->ip_hlen_w;
3088
3089 /* We support checksum offload for TCP and UDP only */
3090 if (xmit_type & XMIT_CSUM_TCP)
3091 hlen += tcp_hdrlen(skb) / 2;
3092 else
3093 hlen += sizeof(struct udphdr) / 2;
3094
3095 pbd->total_hlen_w = cpu_to_le16(hlen);
3096 hlen = hlen*2;
3097
3098 if (xmit_type & XMIT_CSUM_TCP) {
3099 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
3100
3101 } else {
3102 s8 fix = SKB_CS_OFF(skb); /* signed! */
3103
3104 DP(NETIF_MSG_TX_QUEUED,
3105 "hlen %d fix %d csum before fix %x\n",
3106 le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3107
3108 /* HW bug: fixup the CSUM */
3109 pbd->tcp_pseudo_csum =
3110 bnx2x_csum_fix(skb_transport_header(skb),
3111 SKB_CS(skb), fix);
3112
3113 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3114 pbd->tcp_pseudo_csum);
3115 }
3116
3117 return hlen;
3118 }
3119
3120 /* called with netif_tx_lock
3121 * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3122 * netif_wake_queue()
3123 */
3124 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3125 {
3126 struct bnx2x *bp = netdev_priv(dev);
3127
3128 struct netdev_queue *txq;
3129 struct bnx2x_fp_txdata *txdata;
3130 struct sw_tx_bd *tx_buf;
3131 struct eth_tx_start_bd *tx_start_bd, *first_bd;
3132 struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3133 struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3134 struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3135 u32 pbd_e2_parsing_data = 0;
3136 u16 pkt_prod, bd_prod;
3137 int nbd, txq_index;
3138 dma_addr_t mapping;
3139 u32 xmit_type = bnx2x_xmit_type(bp, skb);
3140 int i;
3141 u8 hlen = 0;
3142 __le16 pkt_size = 0;
3143 struct ethhdr *eth;
3144 u8 mac_type = UNICAST_ADDRESS;
3145
3146 #ifdef BNX2X_STOP_ON_ERROR
3147 if (unlikely(bp->panic))
3148 return NETDEV_TX_BUSY;
3149 #endif
3150
3151 txq_index = skb_get_queue_mapping(skb);
3152 txq = netdev_get_tx_queue(dev, txq_index);
3153
3154 BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3155
3156 txdata = &bp->bnx2x_txq[txq_index];
3157
3158 /* enable this debug print to view the transmission queue being used
3159 DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3160 txq_index, fp_index, txdata_index); */
3161
3162 /* enable this debug print to view the tranmission details
3163 DP(NETIF_MSG_TX_QUEUED,
3164 "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3165 txdata->cid, fp_index, txdata_index, txdata, fp); */
3166
3167 if (unlikely(bnx2x_tx_avail(bp, txdata) <
3168 skb_shinfo(skb)->nr_frags +
3169 BDS_PER_TX_PKT +
3170 NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3171 /* Handle special storage cases separately */
3172 if (txdata->tx_ring_size == 0) {
3173 struct bnx2x_eth_q_stats *q_stats =
3174 bnx2x_fp_qstats(bp, txdata->parent_fp);
3175 q_stats->driver_filtered_tx_pkt++;
3176 dev_kfree_skb(skb);
3177 return NETDEV_TX_OK;
3178 }
3179 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3180 netif_tx_stop_queue(txq);
3181 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3182
3183 return NETDEV_TX_BUSY;
3184 }
3185
3186 DP(NETIF_MSG_TX_QUEUED,
3187 "queue[%d]: SKB: summed %x protocol %x protocol(%x,%x) gso type %x xmit_type %x\n",
3188 txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3189 ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
3190
3191 eth = (struct ethhdr *)skb->data;
3192
3193 /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3194 if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3195 if (is_broadcast_ether_addr(eth->h_dest))
3196 mac_type = BROADCAST_ADDRESS;
3197 else
3198 mac_type = MULTICAST_ADDRESS;
3199 }
3200
3201 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3202 /* First, check if we need to linearize the skb (due to FW
3203 restrictions). No need to check fragmentation if page size > 8K
3204 (there will be no violation to FW restrictions) */
3205 if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3206 /* Statistics of linearization */
3207 bp->lin_cnt++;
3208 if (skb_linearize(skb) != 0) {
3209 DP(NETIF_MSG_TX_QUEUED,
3210 "SKB linearization failed - silently dropping this SKB\n");
3211 dev_kfree_skb_any(skb);
3212 return NETDEV_TX_OK;
3213 }
3214 }
3215 #endif
3216 /* Map skb linear data for DMA */
3217 mapping = dma_map_single(&bp->pdev->dev, skb->data,
3218 skb_headlen(skb), DMA_TO_DEVICE);
3219 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3220 DP(NETIF_MSG_TX_QUEUED,
3221 "SKB mapping failed - silently dropping this SKB\n");
3222 dev_kfree_skb_any(skb);
3223 return NETDEV_TX_OK;
3224 }
3225 /*
3226 Please read carefully. First we use one BD which we mark as start,
3227 then we have a parsing info BD (used for TSO or xsum),
3228 and only then we have the rest of the TSO BDs.
3229 (don't forget to mark the last one as last,
3230 and to unmap only AFTER you write to the BD ...)
3231 And above all, all pdb sizes are in words - NOT DWORDS!
3232 */
3233
3234 /* get current pkt produced now - advance it just before sending packet
3235 * since mapping of pages may fail and cause packet to be dropped
3236 */
3237 pkt_prod = txdata->tx_pkt_prod;
3238 bd_prod = TX_BD(txdata->tx_bd_prod);
3239
3240 /* get a tx_buf and first BD
3241 * tx_start_bd may be changed during SPLIT,
3242 * but first_bd will always stay first
3243 */
3244 tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3245 tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3246 first_bd = tx_start_bd;
3247
3248 tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3249 SET_FLAG(tx_start_bd->general_data,
3250 ETH_TX_START_BD_PARSE_NBDS,
3251 0);
3252
3253 /* header nbd */
3254 SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
3255
3256 /* remember the first BD of the packet */
3257 tx_buf->first_bd = txdata->tx_bd_prod;
3258 tx_buf->skb = skb;
3259 tx_buf->flags = 0;
3260
3261 DP(NETIF_MSG_TX_QUEUED,
3262 "sending pkt %u @%p next_idx %u bd %u @%p\n",
3263 pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3264
3265 if (vlan_tx_tag_present(skb)) {
3266 tx_start_bd->vlan_or_ethertype =
3267 cpu_to_le16(vlan_tx_tag_get(skb));
3268 tx_start_bd->bd_flags.as_bitfield |=
3269 (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3270 } else
3271 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3272
3273 /* turn on parsing and get a BD */
3274 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3275
3276 if (xmit_type & XMIT_CSUM)
3277 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3278
3279 if (!CHIP_IS_E1x(bp)) {
3280 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3281 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3282 /* Set PBD in checksum offload case */
3283 if (xmit_type & XMIT_CSUM)
3284 hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3285 &pbd_e2_parsing_data,
3286 xmit_type);
3287 if (IS_MF_SI(bp)) {
3288 /*
3289 * fill in the MAC addresses in the PBD - for local
3290 * switching
3291 */
3292 bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
3293 &pbd_e2->src_mac_addr_mid,
3294 &pbd_e2->src_mac_addr_lo,
3295 eth->h_source);
3296 bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
3297 &pbd_e2->dst_mac_addr_mid,
3298 &pbd_e2->dst_mac_addr_lo,
3299 eth->h_dest);
3300 }
3301
3302 SET_FLAG(pbd_e2_parsing_data,
3303 ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3304 } else {
3305 u16 global_data = 0;
3306 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3307 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3308 /* Set PBD in checksum offload case */
3309 if (xmit_type & XMIT_CSUM)
3310 hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3311
3312 SET_FLAG(global_data,
3313 ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3314 pbd_e1x->global_data |= cpu_to_le16(global_data);
3315 }
3316
3317 /* Setup the data pointer of the first BD of the packet */
3318 tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3319 tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3320 nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3321 tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3322 pkt_size = tx_start_bd->nbytes;
3323
3324 DP(NETIF_MSG_TX_QUEUED,
3325 "first bd @%p addr (%x:%x) nbd %d nbytes %d flags %x vlan %x\n",
3326 tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3327 le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
3328 tx_start_bd->bd_flags.as_bitfield,
3329 le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3330
3331 if (xmit_type & XMIT_GSO) {
3332
3333 DP(NETIF_MSG_TX_QUEUED,
3334 "TSO packet len %d hlen %d total len %d tso size %d\n",
3335 skb->len, hlen, skb_headlen(skb),
3336 skb_shinfo(skb)->gso_size);
3337
3338 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3339
3340 if (unlikely(skb_headlen(skb) > hlen))
3341 bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3342 &tx_start_bd, hlen,
3343 bd_prod, ++nbd);
3344 if (!CHIP_IS_E1x(bp))
3345 bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3346 xmit_type);
3347 else
3348 bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
3349 }
3350
3351 /* Set the PBD's parsing_data field if not zero
3352 * (for the chips newer than 57711).
3353 */
3354 if (pbd_e2_parsing_data)
3355 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3356
3357 tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3358
3359 /* Handle fragmented skb */
3360 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3361 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3362
3363 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3364 skb_frag_size(frag), DMA_TO_DEVICE);
3365 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3366 unsigned int pkts_compl = 0, bytes_compl = 0;
3367
3368 DP(NETIF_MSG_TX_QUEUED,
3369 "Unable to map page - dropping packet...\n");
3370
3371 /* we need unmap all buffers already mapped
3372 * for this SKB;
3373 * first_bd->nbd need to be properly updated
3374 * before call to bnx2x_free_tx_pkt
3375 */
3376 first_bd->nbd = cpu_to_le16(nbd);
3377 bnx2x_free_tx_pkt(bp, txdata,
3378 TX_BD(txdata->tx_pkt_prod),
3379 &pkts_compl, &bytes_compl);
3380 return NETDEV_TX_OK;
3381 }
3382
3383 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3384 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3385 if (total_pkt_bd == NULL)
3386 total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3387
3388 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3389 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3390 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3391 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3392 nbd++;
3393
3394 DP(NETIF_MSG_TX_QUEUED,
3395 "frag %d bd @%p addr (%x:%x) nbytes %d\n",
3396 i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3397 le16_to_cpu(tx_data_bd->nbytes));
3398 }
3399
3400 DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3401
3402 /* update with actual num BDs */
3403 first_bd->nbd = cpu_to_le16(nbd);
3404
3405 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3406
3407 /* now send a tx doorbell, counting the next BD
3408 * if the packet contains or ends with it
3409 */
3410 if (TX_BD_POFF(bd_prod) < nbd)
3411 nbd++;
3412
3413 /* total_pkt_bytes should be set on the first data BD if
3414 * it's not an LSO packet and there is more than one
3415 * data BD. In this case pkt_size is limited by an MTU value.
3416 * However we prefer to set it for an LSO packet (while we don't
3417 * have to) in order to save some CPU cycles in a none-LSO
3418 * case, when we much more care about them.
3419 */
3420 if (total_pkt_bd != NULL)
3421 total_pkt_bd->total_pkt_bytes = pkt_size;
3422
3423 if (pbd_e1x)
3424 DP(NETIF_MSG_TX_QUEUED,
3425 "PBD (E1X) @%p ip_data %x ip_hlen %u ip_id %u lso_mss %u tcp_flags %x xsum %x seq %u hlen %u\n",
3426 pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3427 pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3428 pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3429 le16_to_cpu(pbd_e1x->total_hlen_w));
3430 if (pbd_e2)
3431 DP(NETIF_MSG_TX_QUEUED,
3432 "PBD (E2) @%p dst %x %x %x src %x %x %x parsing_data %x\n",
3433 pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
3434 pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
3435 pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
3436 pbd_e2->parsing_data);
3437 DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d bd %u\n", nbd, bd_prod);
3438
3439 netdev_tx_sent_queue(txq, skb->len);
3440
3441 skb_tx_timestamp(skb);
3442
3443 txdata->tx_pkt_prod++;
3444 /*
3445 * Make sure that the BD data is updated before updating the producer
3446 * since FW might read the BD right after the producer is updated.
3447 * This is only applicable for weak-ordered memory model archs such
3448 * as IA-64. The following barrier is also mandatory since FW will
3449 * assumes packets must have BDs.
3450 */
3451 wmb();
3452
3453 txdata->tx_db.data.prod += nbd;
3454 barrier();
3455
3456 DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3457
3458 mmiowb();
3459
3460 txdata->tx_bd_prod += nbd;
3461
3462 if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3463 netif_tx_stop_queue(txq);
3464
3465 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3466 * ordering of set_bit() in netif_tx_stop_queue() and read of
3467 * fp->bd_tx_cons */
3468 smp_mb();
3469
3470 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3471 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3472 netif_tx_wake_queue(txq);
3473 }
3474 txdata->tx_pkt++;
3475
3476 return NETDEV_TX_OK;
3477 }
3478
3479 /**
3480 * bnx2x_setup_tc - routine to configure net_device for multi tc
3481 *
3482 * @netdev: net device to configure
3483 * @tc: number of traffic classes to enable
3484 *
3485 * callback connected to the ndo_setup_tc function pointer
3486 */
3487 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3488 {
3489 int cos, prio, count, offset;
3490 struct bnx2x *bp = netdev_priv(dev);
3491
3492 /* setup tc must be called under rtnl lock */
3493 ASSERT_RTNL();
3494
3495 /* no traffic classes requested. aborting */
3496 if (!num_tc) {
3497 netdev_reset_tc(dev);
3498 return 0;
3499 }
3500
3501 /* requested to support too many traffic classes */
3502 if (num_tc > bp->max_cos) {
3503 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3504 num_tc, bp->max_cos);
3505 return -EINVAL;
3506 }
3507
3508 /* declare amount of supported traffic classes */
3509 if (netdev_set_num_tc(dev, num_tc)) {
3510 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3511 return -EINVAL;
3512 }
3513
3514 /* configure priority to traffic class mapping */
3515 for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3516 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3517 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3518 "mapping priority %d to tc %d\n",
3519 prio, bp->prio_to_cos[prio]);
3520 }
3521
3522
3523 /* Use this configuration to diffrentiate tc0 from other COSes
3524 This can be used for ets or pfc, and save the effort of setting
3525 up a multio class queue disc or negotiating DCBX with a switch
3526 netdev_set_prio_tc_map(dev, 0, 0);
3527 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3528 for (prio = 1; prio < 16; prio++) {
3529 netdev_set_prio_tc_map(dev, prio, 1);
3530 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3531 } */
3532
3533 /* configure traffic class to transmission queue mapping */
3534 for (cos = 0; cos < bp->max_cos; cos++) {
3535 count = BNX2X_NUM_ETH_QUEUES(bp);
3536 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
3537 netdev_set_tc_queue(dev, cos, count, offset);
3538 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3539 "mapping tc %d to offset %d count %d\n",
3540 cos, offset, count);
3541 }
3542
3543 return 0;
3544 }
3545
3546 /* called with rtnl_lock */
3547 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3548 {
3549 struct sockaddr *addr = p;
3550 struct bnx2x *bp = netdev_priv(dev);
3551 int rc = 0;
3552
3553 if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3554 BNX2X_ERR("Requested MAC address is not valid\n");
3555 return -EINVAL;
3556 }
3557
3558 if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
3559 !is_zero_ether_addr(addr->sa_data)) {
3560 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3561 return -EINVAL;
3562 }
3563
3564 if (netif_running(dev)) {
3565 rc = bnx2x_set_eth_mac(bp, false);
3566 if (rc)
3567 return rc;
3568 }
3569
3570 dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3571 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3572
3573 if (netif_running(dev))
3574 rc = bnx2x_set_eth_mac(bp, true);
3575
3576 return rc;
3577 }
3578
3579 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3580 {
3581 union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3582 struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3583 u8 cos;
3584
3585 /* Common */
3586
3587 if (IS_FCOE_IDX(fp_index)) {
3588 memset(sb, 0, sizeof(union host_hc_status_block));
3589 fp->status_blk_mapping = 0;
3590 } else {
3591 /* status blocks */
3592 if (!CHIP_IS_E1x(bp))
3593 BNX2X_PCI_FREE(sb->e2_sb,
3594 bnx2x_fp(bp, fp_index,
3595 status_blk_mapping),
3596 sizeof(struct host_hc_status_block_e2));
3597 else
3598 BNX2X_PCI_FREE(sb->e1x_sb,
3599 bnx2x_fp(bp, fp_index,
3600 status_blk_mapping),
3601 sizeof(struct host_hc_status_block_e1x));
3602 }
3603
3604 /* Rx */
3605 if (!skip_rx_queue(bp, fp_index)) {
3606 bnx2x_free_rx_bds(fp);
3607
3608 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3609 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3610 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3611 bnx2x_fp(bp, fp_index, rx_desc_mapping),
3612 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3613
3614 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3615 bnx2x_fp(bp, fp_index, rx_comp_mapping),
3616 sizeof(struct eth_fast_path_rx_cqe) *
3617 NUM_RCQ_BD);
3618
3619 /* SGE ring */
3620 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3621 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3622 bnx2x_fp(bp, fp_index, rx_sge_mapping),
3623 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3624 }
3625
3626 /* Tx */
3627 if (!skip_tx_queue(bp, fp_index)) {
3628 /* fastpath tx rings: tx_buf tx_desc */
3629 for_each_cos_in_tx_queue(fp, cos) {
3630 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3631
3632 DP(NETIF_MSG_IFDOWN,
3633 "freeing tx memory of fp %d cos %d cid %d\n",
3634 fp_index, cos, txdata->cid);
3635
3636 BNX2X_FREE(txdata->tx_buf_ring);
3637 BNX2X_PCI_FREE(txdata->tx_desc_ring,
3638 txdata->tx_desc_mapping,
3639 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3640 }
3641 }
3642 /* end of fastpath */
3643 }
3644
3645 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
3646 {
3647 int i;
3648 for_each_cnic_queue(bp, i)
3649 bnx2x_free_fp_mem_at(bp, i);
3650 }
3651
3652 void bnx2x_free_fp_mem(struct bnx2x *bp)
3653 {
3654 int i;
3655 for_each_eth_queue(bp, i)
3656 bnx2x_free_fp_mem_at(bp, i);
3657 }
3658
3659 static void set_sb_shortcuts(struct bnx2x *bp, int index)
3660 {
3661 union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3662 if (!CHIP_IS_E1x(bp)) {
3663 bnx2x_fp(bp, index, sb_index_values) =
3664 (__le16 *)status_blk.e2_sb->sb.index_values;
3665 bnx2x_fp(bp, index, sb_running_index) =
3666 (__le16 *)status_blk.e2_sb->sb.running_index;
3667 } else {
3668 bnx2x_fp(bp, index, sb_index_values) =
3669 (__le16 *)status_blk.e1x_sb->sb.index_values;
3670 bnx2x_fp(bp, index, sb_running_index) =
3671 (__le16 *)status_blk.e1x_sb->sb.running_index;
3672 }
3673 }
3674
3675 /* Returns the number of actually allocated BDs */
3676 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
3677 int rx_ring_size)
3678 {
3679 struct bnx2x *bp = fp->bp;
3680 u16 ring_prod, cqe_ring_prod;
3681 int i, failure_cnt = 0;
3682
3683 fp->rx_comp_cons = 0;
3684 cqe_ring_prod = ring_prod = 0;
3685
3686 /* This routine is called only during fo init so
3687 * fp->eth_q_stats.rx_skb_alloc_failed = 0
3688 */
3689 for (i = 0; i < rx_ring_size; i++) {
3690 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
3691 failure_cnt++;
3692 continue;
3693 }
3694 ring_prod = NEXT_RX_IDX(ring_prod);
3695 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
3696 WARN_ON(ring_prod <= (i - failure_cnt));
3697 }
3698
3699 if (failure_cnt)
3700 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
3701 i - failure_cnt, fp->index);
3702
3703 fp->rx_bd_prod = ring_prod;
3704 /* Limit the CQE producer by the CQE ring size */
3705 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
3706 cqe_ring_prod);
3707 fp->rx_pkt = fp->rx_calls = 0;
3708
3709 bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
3710
3711 return i - failure_cnt;
3712 }
3713
3714 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
3715 {
3716 int i;
3717
3718 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
3719 struct eth_rx_cqe_next_page *nextpg;
3720
3721 nextpg = (struct eth_rx_cqe_next_page *)
3722 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
3723 nextpg->addr_hi =
3724 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
3725 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3726 nextpg->addr_lo =
3727 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
3728 BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3729 }
3730 }
3731
3732 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3733 {
3734 union host_hc_status_block *sb;
3735 struct bnx2x_fastpath *fp = &bp->fp[index];
3736 int ring_size = 0;
3737 u8 cos;
3738 int rx_ring_size = 0;
3739
3740 if (!bp->rx_ring_size &&
3741 (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
3742 rx_ring_size = MIN_RX_SIZE_NONTPA;
3743 bp->rx_ring_size = rx_ring_size;
3744 } else if (!bp->rx_ring_size) {
3745 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3746
3747 if (CHIP_IS_E3(bp)) {
3748 u32 cfg = SHMEM_RD(bp,
3749 dev_info.port_hw_config[BP_PORT(bp)].
3750 default_cfg);
3751
3752 /* Decrease ring size for 1G functions */
3753 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3754 PORT_HW_CFG_NET_SERDES_IF_SGMII)
3755 rx_ring_size /= 10;
3756 }
3757
3758 /* allocate at least number of buffers required by FW */
3759 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3760 MIN_RX_SIZE_TPA, rx_ring_size);
3761
3762 bp->rx_ring_size = rx_ring_size;
3763 } else /* if rx_ring_size specified - use it */
3764 rx_ring_size = bp->rx_ring_size;
3765
3766 /* Common */
3767 sb = &bnx2x_fp(bp, index, status_blk);
3768
3769 if (!IS_FCOE_IDX(index)) {
3770 /* status blocks */
3771 if (!CHIP_IS_E1x(bp))
3772 BNX2X_PCI_ALLOC(sb->e2_sb,
3773 &bnx2x_fp(bp, index, status_blk_mapping),
3774 sizeof(struct host_hc_status_block_e2));
3775 else
3776 BNX2X_PCI_ALLOC(sb->e1x_sb,
3777 &bnx2x_fp(bp, index, status_blk_mapping),
3778 sizeof(struct host_hc_status_block_e1x));
3779 }
3780
3781 /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3782 * set shortcuts for it.
3783 */
3784 if (!IS_FCOE_IDX(index))
3785 set_sb_shortcuts(bp, index);
3786
3787 /* Tx */
3788 if (!skip_tx_queue(bp, index)) {
3789 /* fastpath tx rings: tx_buf tx_desc */
3790 for_each_cos_in_tx_queue(fp, cos) {
3791 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3792
3793 DP(NETIF_MSG_IFUP,
3794 "allocating tx memory of fp %d cos %d\n",
3795 index, cos);
3796
3797 BNX2X_ALLOC(txdata->tx_buf_ring,
3798 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3799 BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3800 &txdata->tx_desc_mapping,
3801 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3802 }
3803 }
3804
3805 /* Rx */
3806 if (!skip_rx_queue(bp, index)) {
3807 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3808 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3809 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3810 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3811 &bnx2x_fp(bp, index, rx_desc_mapping),
3812 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3813
3814 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3815 &bnx2x_fp(bp, index, rx_comp_mapping),
3816 sizeof(struct eth_fast_path_rx_cqe) *
3817 NUM_RCQ_BD);
3818
3819 /* SGE ring */
3820 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3821 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3822 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3823 &bnx2x_fp(bp, index, rx_sge_mapping),
3824 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3825 /* RX BD ring */
3826 bnx2x_set_next_page_rx_bd(fp);
3827
3828 /* CQ ring */
3829 bnx2x_set_next_page_rx_cq(fp);
3830
3831 /* BDs */
3832 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3833 if (ring_size < rx_ring_size)
3834 goto alloc_mem_err;
3835 }
3836
3837 return 0;
3838
3839 /* handles low memory cases */
3840 alloc_mem_err:
3841 BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3842 index, ring_size);
3843 /* FW will drop all packets if queue is not big enough,
3844 * In these cases we disable the queue
3845 * Min size is different for OOO, TPA and non-TPA queues
3846 */
3847 if (ring_size < (fp->disable_tpa ?
3848 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3849 /* release memory allocated for this queue */
3850 bnx2x_free_fp_mem_at(bp, index);
3851 return -ENOMEM;
3852 }
3853 return 0;
3854 }
3855
3856 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
3857 {
3858 if (!NO_FCOE(bp))
3859 /* FCoE */
3860 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
3861 /* we will fail load process instead of mark
3862 * NO_FCOE_FLAG
3863 */
3864 return -ENOMEM;
3865
3866 return 0;
3867 }
3868
3869 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3870 {
3871 int i;
3872
3873 /* 1. Allocate FP for leading - fatal if error
3874 * 2. Allocate RSS - fix number of queues if error
3875 */
3876
3877 /* leading */
3878 if (bnx2x_alloc_fp_mem_at(bp, 0))
3879 return -ENOMEM;
3880
3881 /* RSS */
3882 for_each_nondefault_eth_queue(bp, i)
3883 if (bnx2x_alloc_fp_mem_at(bp, i))
3884 break;
3885
3886 /* handle memory failures */
3887 if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3888 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3889
3890 WARN_ON(delta < 0);
3891 bnx2x_shrink_eth_fp(bp, delta);
3892 if (CNIC_SUPPORT(bp))
3893 /* move non eth FPs next to last eth FP
3894 * must be done in that order
3895 * FCOE_IDX < FWD_IDX < OOO_IDX
3896 */
3897
3898 /* move FCoE fp even NO_FCOE_FLAG is on */
3899 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
3900 bp->num_ethernet_queues -= delta;
3901 bp->num_queues = bp->num_ethernet_queues +
3902 bp->num_cnic_queues;
3903 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3904 bp->num_queues + delta, bp->num_queues);
3905 }
3906
3907 return 0;
3908 }
3909
3910 void bnx2x_free_mem_bp(struct bnx2x *bp)
3911 {
3912 kfree(bp->fp->tpa_info);
3913 kfree(bp->fp);
3914 kfree(bp->sp_objs);
3915 kfree(bp->fp_stats);
3916 kfree(bp->bnx2x_txq);
3917 kfree(bp->msix_table);
3918 kfree(bp->ilt);
3919 }
3920
3921 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
3922 {
3923 struct bnx2x_fastpath *fp;
3924 struct msix_entry *tbl;
3925 struct bnx2x_ilt *ilt;
3926 int msix_table_size = 0;
3927 int fp_array_size, txq_array_size;
3928 int i;
3929
3930 /*
3931 * The biggest MSI-X table we might need is as a maximum number of fast
3932 * path IGU SBs plus default SB (for PF).
3933 */
3934 msix_table_size = bp->igu_sb_cnt + 1;
3935
3936 /* fp array: RSS plus CNIC related L2 queues */
3937 fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
3938 BNX2X_DEV_INFO("fp_array_size %d", fp_array_size);
3939
3940 fp = kcalloc(fp_array_size, sizeof(*fp), GFP_KERNEL);
3941 if (!fp)
3942 goto alloc_err;
3943 for (i = 0; i < fp_array_size; i++) {
3944 fp[i].tpa_info =
3945 kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
3946 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
3947 if (!(fp[i].tpa_info))
3948 goto alloc_err;
3949 }
3950
3951 bp->fp = fp;
3952
3953 /* allocate sp objs */
3954 bp->sp_objs = kcalloc(fp_array_size, sizeof(struct bnx2x_sp_objs),
3955 GFP_KERNEL);
3956 if (!bp->sp_objs)
3957 goto alloc_err;
3958
3959 /* allocate fp_stats */
3960 bp->fp_stats = kcalloc(fp_array_size, sizeof(struct bnx2x_fp_stats),
3961 GFP_KERNEL);
3962 if (!bp->fp_stats)
3963 goto alloc_err;
3964
3965 /* Allocate memory for the transmission queues array */
3966 txq_array_size =
3967 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
3968 BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
3969
3970 bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
3971 GFP_KERNEL);
3972 if (!bp->bnx2x_txq)
3973 goto alloc_err;
3974
3975 /* msix table */
3976 tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3977 if (!tbl)
3978 goto alloc_err;
3979 bp->msix_table = tbl;
3980
3981 /* ilt */
3982 ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3983 if (!ilt)
3984 goto alloc_err;
3985 bp->ilt = ilt;
3986
3987 return 0;
3988 alloc_err:
3989 bnx2x_free_mem_bp(bp);
3990 return -ENOMEM;
3991
3992 }
3993
3994 int bnx2x_reload_if_running(struct net_device *dev)
3995 {
3996 struct bnx2x *bp = netdev_priv(dev);
3997
3998 if (unlikely(!netif_running(dev)))
3999 return 0;
4000
4001 bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4002 return bnx2x_nic_load(bp, LOAD_NORMAL);
4003 }
4004
4005 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4006 {
4007 u32 sel_phy_idx = 0;
4008 if (bp->link_params.num_phys <= 1)
4009 return INT_PHY;
4010
4011 if (bp->link_vars.link_up) {
4012 sel_phy_idx = EXT_PHY1;
4013 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4014 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4015 (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4016 sel_phy_idx = EXT_PHY2;
4017 } else {
4018
4019 switch (bnx2x_phy_selection(&bp->link_params)) {
4020 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4021 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4022 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4023 sel_phy_idx = EXT_PHY1;
4024 break;
4025 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4026 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4027 sel_phy_idx = EXT_PHY2;
4028 break;
4029 }
4030 }
4031
4032 return sel_phy_idx;
4033
4034 }
4035 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4036 {
4037 u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4038 /*
4039 * The selected actived PHY is always after swapping (in case PHY
4040 * swapping is enabled). So when swapping is enabled, we need to reverse
4041 * the configuration
4042 */
4043
4044 if (bp->link_params.multi_phy_config &
4045 PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4046 if (sel_phy_idx == EXT_PHY1)
4047 sel_phy_idx = EXT_PHY2;
4048 else if (sel_phy_idx == EXT_PHY2)
4049 sel_phy_idx = EXT_PHY1;
4050 }
4051 return LINK_CONFIG_IDX(sel_phy_idx);
4052 }
4053
4054 #ifdef NETDEV_FCOE_WWNN
4055 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4056 {
4057 struct bnx2x *bp = netdev_priv(dev);
4058 struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4059
4060 switch (type) {
4061 case NETDEV_FCOE_WWNN:
4062 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4063 cp->fcoe_wwn_node_name_lo);
4064 break;
4065 case NETDEV_FCOE_WWPN:
4066 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4067 cp->fcoe_wwn_port_name_lo);
4068 break;
4069 default:
4070 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4071 return -EINVAL;
4072 }
4073
4074 return 0;
4075 }
4076 #endif
4077
4078 /* called with rtnl_lock */
4079 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4080 {
4081 struct bnx2x *bp = netdev_priv(dev);
4082
4083 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4084 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4085 return -EAGAIN;
4086 }
4087
4088 if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4089 ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4090 BNX2X_ERR("Can't support requested MTU size\n");
4091 return -EINVAL;
4092 }
4093
4094 /* This does not race with packet allocation
4095 * because the actual alloc size is
4096 * only updated as part of load
4097 */
4098 dev->mtu = new_mtu;
4099
4100 return bnx2x_reload_if_running(dev);
4101 }
4102
4103 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4104 netdev_features_t features)
4105 {
4106 struct bnx2x *bp = netdev_priv(dev);
4107
4108 /* TPA requires Rx CSUM offloading */
4109 if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4110 features &= ~NETIF_F_LRO;
4111 features &= ~NETIF_F_GRO;
4112 }
4113
4114 return features;
4115 }
4116
4117 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4118 {
4119 struct bnx2x *bp = netdev_priv(dev);
4120 u32 flags = bp->flags;
4121 bool bnx2x_reload = false;
4122
4123 if (features & NETIF_F_LRO)
4124 flags |= TPA_ENABLE_FLAG;
4125 else
4126 flags &= ~TPA_ENABLE_FLAG;
4127
4128 if (features & NETIF_F_GRO)
4129 flags |= GRO_ENABLE_FLAG;
4130 else
4131 flags &= ~GRO_ENABLE_FLAG;
4132
4133 if (features & NETIF_F_LOOPBACK) {
4134 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4135 bp->link_params.loopback_mode = LOOPBACK_BMAC;
4136 bnx2x_reload = true;
4137 }
4138 } else {
4139 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4140 bp->link_params.loopback_mode = LOOPBACK_NONE;
4141 bnx2x_reload = true;
4142 }
4143 }
4144
4145 if (flags ^ bp->flags) {
4146 bp->flags = flags;
4147 bnx2x_reload = true;
4148 }
4149
4150 if (bnx2x_reload) {
4151 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4152 return bnx2x_reload_if_running(dev);
4153 /* else: bnx2x_nic_load() will be called at end of recovery */
4154 }
4155
4156 return 0;
4157 }
4158
4159 void bnx2x_tx_timeout(struct net_device *dev)
4160 {
4161 struct bnx2x *bp = netdev_priv(dev);
4162
4163 #ifdef BNX2X_STOP_ON_ERROR
4164 if (!bp->panic)
4165 bnx2x_panic();
4166 #endif
4167
4168 smp_mb__before_clear_bit();
4169 set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4170 smp_mb__after_clear_bit();
4171
4172 /* This allows the netif to be shutdown gracefully before resetting */
4173 schedule_delayed_work(&bp->sp_rtnl_task, 0);
4174 }
4175
4176 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4177 {
4178 struct net_device *dev = pci_get_drvdata(pdev);
4179 struct bnx2x *bp;
4180
4181 if (!dev) {
4182 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4183 return -ENODEV;
4184 }
4185 bp = netdev_priv(dev);
4186
4187 rtnl_lock();
4188
4189 pci_save_state(pdev);
4190
4191 if (!netif_running(dev)) {
4192 rtnl_unlock();
4193 return 0;
4194 }
4195
4196 netif_device_detach(dev);
4197
4198 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4199
4200 bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4201
4202 rtnl_unlock();
4203
4204 return 0;
4205 }
4206
4207 int bnx2x_resume(struct pci_dev *pdev)
4208 {
4209 struct net_device *dev = pci_get_drvdata(pdev);
4210 struct bnx2x *bp;
4211 int rc;
4212
4213 if (!dev) {
4214 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4215 return -ENODEV;
4216 }
4217 bp = netdev_priv(dev);
4218
4219 if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4220 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4221 return -EAGAIN;
4222 }
4223
4224 rtnl_lock();
4225
4226 pci_restore_state(pdev);
4227
4228 if (!netif_running(dev)) {
4229 rtnl_unlock();
4230 return 0;
4231 }
4232
4233 bnx2x_set_power_state(bp, PCI_D0);
4234 netif_device_attach(dev);
4235
4236 rc = bnx2x_nic_load(bp, LOAD_OPEN);
4237
4238 rtnl_unlock();
4239
4240 return rc;
4241 }
4242
4243
4244 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4245 u32 cid)
4246 {
4247 /* ustorm cxt validation */
4248 cxt->ustorm_ag_context.cdu_usage =
4249 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4250 CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4251 /* xcontext validation */
4252 cxt->xstorm_ag_context.cdu_reserved =
4253 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4254 CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4255 }
4256
4257 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4258 u8 fw_sb_id, u8 sb_index,
4259 u8 ticks)
4260 {
4261
4262 u32 addr = BAR_CSTRORM_INTMEM +
4263 CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4264 REG_WR8(bp, addr, ticks);
4265 DP(NETIF_MSG_IFUP,
4266 "port %x fw_sb_id %d sb_index %d ticks %d\n",
4267 port, fw_sb_id, sb_index, ticks);
4268 }
4269
4270 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4271 u16 fw_sb_id, u8 sb_index,
4272 u8 disable)
4273 {
4274 u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4275 u32 addr = BAR_CSTRORM_INTMEM +
4276 CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4277 u16 flags = REG_RD16(bp, addr);
4278 /* clear and set */
4279 flags &= ~HC_INDEX_DATA_HC_ENABLED;
4280 flags |= enable_flag;
4281 REG_WR16(bp, addr, flags);
4282 DP(NETIF_MSG_IFUP,
4283 "port %x fw_sb_id %d sb_index %d disable %d\n",
4284 port, fw_sb_id, sb_index, disable);
4285 }
4286
4287 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4288 u8 sb_index, u8 disable, u16 usec)
4289 {
4290 int port = BP_PORT(bp);
4291 u8 ticks = usec / BNX2X_BTR;
4292
4293 storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4294
4295 disable = disable ? 1 : (usec ? 0 : 1);
4296 storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4297 }
kernel source for telekom puls (alcatel/mediatek)