1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2011 Solarflare Communications Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
11 #include <linux/socket.h>
13 #include <linux/slab.h>
15 #include <linux/tcp.h>
16 #include <linux/udp.h>
17 #include <linux/prefetch.h>
18 #include <linux/moduleparam.h>
20 #include <net/checksum.h>
21 #include "net_driver.h"
25 #include "workarounds.h"
27 /* Number of RX descriptors pushed at once. */
28 #define EFX_RX_BATCH 8
30 /* Maximum size of a buffer sharing a page */
31 #define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state))
33 /* Size of buffer allocated for skb header area. */
34 #define EFX_SKB_HEADERS 64u
37 * rx_alloc_method - RX buffer allocation method
39 * This driver supports two methods for allocating and using RX buffers:
40 * each RX buffer may be backed by an skb or by an order-n page.
42 * When GRO is in use then the second method has a lower overhead,
43 * since we don't have to allocate then free skbs on reassembled frames.
46 * - RX_ALLOC_METHOD_AUTO = 0
47 * - RX_ALLOC_METHOD_SKB = 1
48 * - RX_ALLOC_METHOD_PAGE = 2
50 * The heuristic for %RX_ALLOC_METHOD_AUTO is a simple hysteresis count
51 * controlled by the parameters below.
53 * - Since pushing and popping descriptors are separated by the rx_queue
54 * size, so the watermarks should be ~rxd_size.
55 * - The performance win by using page-based allocation for GRO is less
56 * than the performance hit of using page-based allocation of non-GRO,
57 * so the watermarks should reflect this.
59 * Per channel we maintain a single variable, updated by each channel:
61 * rx_alloc_level += (gro_performed ? RX_ALLOC_FACTOR_GRO :
62 * RX_ALLOC_FACTOR_SKB)
63 * Per NAPI poll interval, we constrain rx_alloc_level to 0..MAX (which
64 * limits the hysteresis), and update the allocation strategy:
66 * rx_alloc_method = (rx_alloc_level > RX_ALLOC_LEVEL_GRO ?
67 * RX_ALLOC_METHOD_PAGE : RX_ALLOC_METHOD_SKB)
69 static int rx_alloc_method
= RX_ALLOC_METHOD_AUTO
;
71 #define RX_ALLOC_LEVEL_GRO 0x2000
72 #define RX_ALLOC_LEVEL_MAX 0x3000
73 #define RX_ALLOC_FACTOR_GRO 1
74 #define RX_ALLOC_FACTOR_SKB (-2)
76 /* This is the percentage fill level below which new RX descriptors
77 * will be added to the RX descriptor ring.
79 static unsigned int rx_refill_threshold
;
82 * RX maximum head room required.
84 * This must be at least 1 to prevent overflow and at least 2 to allow
87 #define EFX_RXD_HEAD_ROOM 2
89 /* Offset of ethernet header within page */
90 static inline unsigned int efx_rx_buf_offset(struct efx_nic
*efx
,
91 struct efx_rx_buffer
*buf
)
93 /* Offset is always within one page, so we don't need to consider
96 return ((unsigned int) buf
->dma_addr
& (PAGE_SIZE
- 1)) +
97 efx
->type
->rx_buffer_hash_size
;
99 static inline unsigned int efx_rx_buf_size(struct efx_nic
*efx
)
101 return PAGE_SIZE
<< efx
->rx_buffer_order
;
104 static u8
*efx_rx_buf_eh(struct efx_nic
*efx
, struct efx_rx_buffer
*buf
)
106 if (buf
->flags
& EFX_RX_BUF_PAGE
)
107 return page_address(buf
->u
.page
) + efx_rx_buf_offset(efx
, buf
);
109 return (u8
*)buf
->u
.skb
->data
+ efx
->type
->rx_buffer_hash_size
;
112 static inline u32
efx_rx_buf_hash(const u8
*eh
)
114 /* The ethernet header is always directly after any hash. */
115 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0
116 return __le32_to_cpup((const __le32
*)(eh
- 4));
118 const u8
*data
= eh
- 4;
119 return (u32
)data
[0] |
127 * efx_init_rx_buffers_skb - create EFX_RX_BATCH skb-based RX buffers
129 * @rx_queue: Efx RX queue
131 * This allocates EFX_RX_BATCH skbs, maps them for DMA, and populates a
132 * struct efx_rx_buffer for each one. Return a negative error code or 0
133 * on success. May fail having only inserted fewer than EFX_RX_BATCH
136 static int efx_init_rx_buffers_skb(struct efx_rx_queue
*rx_queue
)
138 struct efx_nic
*efx
= rx_queue
->efx
;
139 struct net_device
*net_dev
= efx
->net_dev
;
140 struct efx_rx_buffer
*rx_buf
;
142 int skb_len
= efx
->rx_buffer_len
;
143 unsigned index
, count
;
145 for (count
= 0; count
< EFX_RX_BATCH
; ++count
) {
146 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
147 rx_buf
= efx_rx_buffer(rx_queue
, index
);
149 rx_buf
->u
.skb
= skb
= netdev_alloc_skb(net_dev
, skb_len
);
153 /* Adjust the SKB for padding */
154 skb_reserve(skb
, NET_IP_ALIGN
);
155 rx_buf
->len
= skb_len
- NET_IP_ALIGN
;
158 rx_buf
->dma_addr
= dma_map_single(&efx
->pci_dev
->dev
,
159 skb
->data
, rx_buf
->len
,
161 if (unlikely(dma_mapping_error(&efx
->pci_dev
->dev
,
162 rx_buf
->dma_addr
))) {
163 dev_kfree_skb_any(skb
);
164 rx_buf
->u
.skb
= NULL
;
168 ++rx_queue
->added_count
;
169 ++rx_queue
->alloc_skb_count
;
176 * efx_init_rx_buffers_page - create EFX_RX_BATCH page-based RX buffers
178 * @rx_queue: Efx RX queue
180 * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA,
181 * and populates struct efx_rx_buffers for each one. Return a negative error
182 * code or 0 on success. If a single page can be split between two buffers,
183 * then the page will either be inserted fully, or not at at all.
185 static int efx_init_rx_buffers_page(struct efx_rx_queue
*rx_queue
)
187 struct efx_nic
*efx
= rx_queue
->efx
;
188 struct efx_rx_buffer
*rx_buf
;
191 struct efx_rx_page_state
*state
;
193 unsigned index
, count
;
195 /* We can split a page between two buffers */
196 BUILD_BUG_ON(EFX_RX_BATCH
& 1);
198 for (count
= 0; count
< EFX_RX_BATCH
; ++count
) {
199 page
= alloc_pages(__GFP_COLD
| __GFP_COMP
| GFP_ATOMIC
,
200 efx
->rx_buffer_order
);
201 if (unlikely(page
== NULL
))
203 dma_addr
= dma_map_page(&efx
->pci_dev
->dev
, page
, 0,
204 efx_rx_buf_size(efx
),
206 if (unlikely(dma_mapping_error(&efx
->pci_dev
->dev
, dma_addr
))) {
207 __free_pages(page
, efx
->rx_buffer_order
);
210 page_addr
= page_address(page
);
213 state
->dma_addr
= dma_addr
;
215 page_addr
+= sizeof(struct efx_rx_page_state
);
216 dma_addr
+= sizeof(struct efx_rx_page_state
);
219 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
220 rx_buf
= efx_rx_buffer(rx_queue
, index
);
221 rx_buf
->dma_addr
= dma_addr
+ EFX_PAGE_IP_ALIGN
;
222 rx_buf
->u
.page
= page
;
223 rx_buf
->len
= efx
->rx_buffer_len
- EFX_PAGE_IP_ALIGN
;
224 rx_buf
->flags
= EFX_RX_BUF_PAGE
;
225 ++rx_queue
->added_count
;
226 ++rx_queue
->alloc_page_count
;
229 if ((~count
& 1) && (efx
->rx_buffer_len
<= EFX_RX_HALF_PAGE
)) {
230 /* Use the second half of the page */
232 dma_addr
+= (PAGE_SIZE
>> 1);
233 page_addr
+= (PAGE_SIZE
>> 1);
242 static void efx_unmap_rx_buffer(struct efx_nic
*efx
,
243 struct efx_rx_buffer
*rx_buf
)
245 if ((rx_buf
->flags
& EFX_RX_BUF_PAGE
) && rx_buf
->u
.page
) {
246 struct efx_rx_page_state
*state
;
248 state
= page_address(rx_buf
->u
.page
);
249 if (--state
->refcnt
== 0) {
250 dma_unmap_page(&efx
->pci_dev
->dev
,
252 efx_rx_buf_size(efx
),
255 } else if (!(rx_buf
->flags
& EFX_RX_BUF_PAGE
) && rx_buf
->u
.skb
) {
256 dma_unmap_single(&efx
->pci_dev
->dev
, rx_buf
->dma_addr
,
257 rx_buf
->len
, DMA_FROM_DEVICE
);
261 static void efx_free_rx_buffer(struct efx_nic
*efx
,
262 struct efx_rx_buffer
*rx_buf
)
264 if ((rx_buf
->flags
& EFX_RX_BUF_PAGE
) && rx_buf
->u
.page
) {
265 __free_pages(rx_buf
->u
.page
, efx
->rx_buffer_order
);
266 rx_buf
->u
.page
= NULL
;
267 } else if (!(rx_buf
->flags
& EFX_RX_BUF_PAGE
) && rx_buf
->u
.skb
) {
268 dev_kfree_skb_any(rx_buf
->u
.skb
);
269 rx_buf
->u
.skb
= NULL
;
273 static void efx_fini_rx_buffer(struct efx_rx_queue
*rx_queue
,
274 struct efx_rx_buffer
*rx_buf
)
276 efx_unmap_rx_buffer(rx_queue
->efx
, rx_buf
);
277 efx_free_rx_buffer(rx_queue
->efx
, rx_buf
);
280 /* Attempt to resurrect the other receive buffer that used to share this page,
281 * which had previously been passed up to the kernel and freed. */
282 static void efx_resurrect_rx_buffer(struct efx_rx_queue
*rx_queue
,
283 struct efx_rx_buffer
*rx_buf
)
285 struct efx_rx_page_state
*state
= page_address(rx_buf
->u
.page
);
286 struct efx_rx_buffer
*new_buf
;
287 unsigned fill_level
, index
;
289 /* +1 because efx_rx_packet() incremented removed_count. +1 because
290 * we'd like to insert an additional descriptor whilst leaving
291 * EFX_RXD_HEAD_ROOM for the non-recycle path */
292 fill_level
= (rx_queue
->added_count
- rx_queue
->removed_count
+ 2);
293 if (unlikely(fill_level
> rx_queue
->max_fill
)) {
294 /* We could place "state" on a list, and drain the list in
295 * efx_fast_push_rx_descriptors(). For now, this will do. */
300 get_page(rx_buf
->u
.page
);
302 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
303 new_buf
= efx_rx_buffer(rx_queue
, index
);
304 new_buf
->dma_addr
= rx_buf
->dma_addr
^ (PAGE_SIZE
>> 1);
305 new_buf
->u
.page
= rx_buf
->u
.page
;
306 new_buf
->len
= rx_buf
->len
;
307 new_buf
->flags
= EFX_RX_BUF_PAGE
;
308 ++rx_queue
->added_count
;
311 /* Recycle the given rx buffer directly back into the rx_queue. There is
312 * always room to add this buffer, because we've just popped a buffer. */
313 static void efx_recycle_rx_buffer(struct efx_channel
*channel
,
314 struct efx_rx_buffer
*rx_buf
)
316 struct efx_nic
*efx
= channel
->efx
;
317 struct efx_rx_queue
*rx_queue
= efx_channel_get_rx_queue(channel
);
318 struct efx_rx_buffer
*new_buf
;
321 rx_buf
->flags
&= EFX_RX_BUF_PAGE
;
323 if ((rx_buf
->flags
& EFX_RX_BUF_PAGE
) &&
324 efx
->rx_buffer_len
<= EFX_RX_HALF_PAGE
&&
325 page_count(rx_buf
->u
.page
) == 1)
326 efx_resurrect_rx_buffer(rx_queue
, rx_buf
);
328 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
329 new_buf
= efx_rx_buffer(rx_queue
, index
);
331 memcpy(new_buf
, rx_buf
, sizeof(*new_buf
));
332 rx_buf
->u
.page
= NULL
;
333 ++rx_queue
->added_count
;
337 * efx_fast_push_rx_descriptors - push new RX descriptors quickly
338 * @rx_queue: RX descriptor queue
340 * This will aim to fill the RX descriptor queue up to
341 * @rx_queue->@max_fill. If there is insufficient atomic
342 * memory to do so, a slow fill will be scheduled.
344 * The caller must provide serialisation (none is used here). In practise,
345 * this means this function must run from the NAPI handler, or be called
346 * when NAPI is disabled.
348 void efx_fast_push_rx_descriptors(struct efx_rx_queue
*rx_queue
)
350 struct efx_channel
*channel
= efx_rx_queue_channel(rx_queue
);
354 /* Calculate current fill level, and exit if we don't need to fill */
355 fill_level
= (rx_queue
->added_count
- rx_queue
->removed_count
);
356 EFX_BUG_ON_PARANOID(fill_level
> rx_queue
->efx
->rxq_entries
);
357 if (fill_level
>= rx_queue
->fast_fill_trigger
)
360 /* Record minimum fill level */
361 if (unlikely(fill_level
< rx_queue
->min_fill
)) {
363 rx_queue
->min_fill
= fill_level
;
366 space
= rx_queue
->max_fill
- fill_level
;
367 EFX_BUG_ON_PARANOID(space
< EFX_RX_BATCH
);
369 netif_vdbg(rx_queue
->efx
, rx_status
, rx_queue
->efx
->net_dev
,
370 "RX queue %d fast-filling descriptor ring from"
371 " level %d to level %d using %s allocation\n",
372 efx_rx_queue_index(rx_queue
), fill_level
,
374 channel
->rx_alloc_push_pages
? "page" : "skb");
377 if (channel
->rx_alloc_push_pages
)
378 rc
= efx_init_rx_buffers_page(rx_queue
);
380 rc
= efx_init_rx_buffers_skb(rx_queue
);
382 /* Ensure that we don't leave the rx queue empty */
383 if (rx_queue
->added_count
== rx_queue
->removed_count
)
384 efx_schedule_slow_fill(rx_queue
);
387 } while ((space
-= EFX_RX_BATCH
) >= EFX_RX_BATCH
);
389 netif_vdbg(rx_queue
->efx
, rx_status
, rx_queue
->efx
->net_dev
,
390 "RX queue %d fast-filled descriptor ring "
391 "to level %d\n", efx_rx_queue_index(rx_queue
),
392 rx_queue
->added_count
- rx_queue
->removed_count
);
395 if (rx_queue
->notified_count
!= rx_queue
->added_count
)
396 efx_nic_notify_rx_desc(rx_queue
);
399 void efx_rx_slow_fill(unsigned long context
)
401 struct efx_rx_queue
*rx_queue
= (struct efx_rx_queue
*)context
;
403 /* Post an event to cause NAPI to run and refill the queue */
404 efx_nic_generate_fill_event(rx_queue
);
405 ++rx_queue
->slow_fill_count
;
408 static void efx_rx_packet__check_len(struct efx_rx_queue
*rx_queue
,
409 struct efx_rx_buffer
*rx_buf
,
410 int len
, bool *leak_packet
)
412 struct efx_nic
*efx
= rx_queue
->efx
;
413 unsigned max_len
= rx_buf
->len
- efx
->type
->rx_buffer_padding
;
415 if (likely(len
<= max_len
))
418 /* The packet must be discarded, but this is only a fatal error
419 * if the caller indicated it was
421 rx_buf
->flags
|= EFX_RX_PKT_DISCARD
;
423 if ((len
> rx_buf
->len
) && EFX_WORKAROUND_8071(efx
)) {
425 netif_err(efx
, rx_err
, efx
->net_dev
,
426 " RX queue %d seriously overlength "
427 "RX event (0x%x > 0x%x+0x%x). Leaking\n",
428 efx_rx_queue_index(rx_queue
), len
, max_len
,
429 efx
->type
->rx_buffer_padding
);
430 /* If this buffer was skb-allocated, then the meta
431 * data at the end of the skb will be trashed. So
432 * we have no choice but to leak the fragment.
434 *leak_packet
= !(rx_buf
->flags
& EFX_RX_BUF_PAGE
);
435 efx_schedule_reset(efx
, RESET_TYPE_RX_RECOVERY
);
438 netif_err(efx
, rx_err
, efx
->net_dev
,
439 " RX queue %d overlength RX event "
441 efx_rx_queue_index(rx_queue
), len
, max_len
);
444 efx_rx_queue_channel(rx_queue
)->n_rx_overlength
++;
447 /* Pass a received packet up through GRO. GRO can handle pages
448 * regardless of checksum state and skbs with a good checksum.
450 static void efx_rx_packet_gro(struct efx_channel
*channel
,
451 struct efx_rx_buffer
*rx_buf
,
454 struct napi_struct
*napi
= &channel
->napi_str
;
455 gro_result_t gro_result
;
457 if (rx_buf
->flags
& EFX_RX_BUF_PAGE
) {
458 struct efx_nic
*efx
= channel
->efx
;
459 struct page
*page
= rx_buf
->u
.page
;
462 rx_buf
->u
.page
= NULL
;
464 skb
= napi_get_frags(napi
);
470 if (efx
->net_dev
->features
& NETIF_F_RXHASH
)
471 skb
->rxhash
= efx_rx_buf_hash(eh
);
473 skb_fill_page_desc(skb
, 0, page
,
474 efx_rx_buf_offset(efx
, rx_buf
), rx_buf
->len
);
476 skb
->len
= rx_buf
->len
;
477 skb
->data_len
= rx_buf
->len
;
478 skb
->truesize
+= rx_buf
->len
;
479 skb
->ip_summed
= ((rx_buf
->flags
& EFX_RX_PKT_CSUMMED
) ?
480 CHECKSUM_UNNECESSARY
: CHECKSUM_NONE
);
482 skb_record_rx_queue(skb
, channel
->channel
);
484 gro_result
= napi_gro_frags(napi
);
486 struct sk_buff
*skb
= rx_buf
->u
.skb
;
488 EFX_BUG_ON_PARANOID(!(rx_buf
->flags
& EFX_RX_PKT_CSUMMED
));
489 rx_buf
->u
.skb
= NULL
;
490 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
492 gro_result
= napi_gro_receive(napi
, skb
);
495 if (gro_result
== GRO_NORMAL
) {
496 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_SKB
;
497 } else if (gro_result
!= GRO_DROP
) {
498 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_GRO
;
499 channel
->irq_mod_score
+= 2;
503 void efx_rx_packet(struct efx_rx_queue
*rx_queue
, unsigned int index
,
504 unsigned int len
, u16 flags
)
506 struct efx_nic
*efx
= rx_queue
->efx
;
507 struct efx_channel
*channel
= efx_rx_queue_channel(rx_queue
);
508 struct efx_rx_buffer
*rx_buf
;
509 bool leak_packet
= false;
511 rx_buf
= efx_rx_buffer(rx_queue
, index
);
512 rx_buf
->flags
|= flags
;
514 /* This allows the refill path to post another buffer.
515 * EFX_RXD_HEAD_ROOM ensures that the slot we are using
516 * isn't overwritten yet.
518 rx_queue
->removed_count
++;
520 /* Validate the length encoded in the event vs the descriptor pushed */
521 efx_rx_packet__check_len(rx_queue
, rx_buf
, len
, &leak_packet
);
523 netif_vdbg(efx
, rx_status
, efx
->net_dev
,
524 "RX queue %d received id %x at %llx+%x %s%s\n",
525 efx_rx_queue_index(rx_queue
), index
,
526 (unsigned long long)rx_buf
->dma_addr
, len
,
527 (rx_buf
->flags
& EFX_RX_PKT_CSUMMED
) ? " [SUMMED]" : "",
528 (rx_buf
->flags
& EFX_RX_PKT_DISCARD
) ? " [DISCARD]" : "");
530 /* Discard packet, if instructed to do so */
531 if (unlikely(rx_buf
->flags
& EFX_RX_PKT_DISCARD
)) {
532 if (unlikely(leak_packet
))
533 channel
->n_skbuff_leaks
++;
535 efx_recycle_rx_buffer(channel
, rx_buf
);
537 /* Don't hold off the previous receive */
542 /* Release card resources - assumes all RX buffers consumed in-order
545 efx_unmap_rx_buffer(efx
, rx_buf
);
547 /* Prefetch nice and early so data will (hopefully) be in cache by
548 * the time we look at it.
550 prefetch(efx_rx_buf_eh(efx
, rx_buf
));
552 /* Pipeline receives so that we give time for packet headers to be
553 * prefetched into cache.
555 rx_buf
->len
= len
- efx
->type
->rx_buffer_hash_size
;
558 __efx_rx_packet(channel
, channel
->rx_pkt
);
559 channel
->rx_pkt
= rx_buf
;
562 static void efx_rx_deliver(struct efx_channel
*channel
,
563 struct efx_rx_buffer
*rx_buf
)
567 /* We now own the SKB */
569 rx_buf
->u
.skb
= NULL
;
571 /* Set the SKB flags */
572 skb_checksum_none_assert(skb
);
574 /* Pass the packet up */
575 netif_receive_skb(skb
);
577 /* Update allocation strategy method */
578 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_SKB
;
581 /* Handle a received packet. Second half: Touches packet payload. */
582 void __efx_rx_packet(struct efx_channel
*channel
, struct efx_rx_buffer
*rx_buf
)
584 struct efx_nic
*efx
= channel
->efx
;
585 u8
*eh
= efx_rx_buf_eh(efx
, rx_buf
);
587 /* If we're in loopback test, then pass the packet directly to the
588 * loopback layer, and free the rx_buf here
590 if (unlikely(efx
->loopback_selftest
)) {
591 efx_loopback_rx_packet(efx
, eh
, rx_buf
->len
);
592 efx_free_rx_buffer(efx
, rx_buf
);
596 if (!(rx_buf
->flags
& EFX_RX_BUF_PAGE
)) {
597 struct sk_buff
*skb
= rx_buf
->u
.skb
;
599 prefetch(skb_shinfo(skb
));
601 skb_reserve(skb
, efx
->type
->rx_buffer_hash_size
);
602 skb_put(skb
, rx_buf
->len
);
604 if (efx
->net_dev
->features
& NETIF_F_RXHASH
)
605 skb
->rxhash
= efx_rx_buf_hash(eh
);
607 /* Move past the ethernet header. rx_buf->data still points
608 * at the ethernet header */
609 skb
->protocol
= eth_type_trans(skb
, efx
->net_dev
);
611 skb_record_rx_queue(skb
, channel
->channel
);
614 if (unlikely(!(efx
->net_dev
->features
& NETIF_F_RXCSUM
)))
615 rx_buf
->flags
&= ~EFX_RX_PKT_CSUMMED
;
617 if (likely(rx_buf
->flags
& (EFX_RX_BUF_PAGE
| EFX_RX_PKT_CSUMMED
)))
618 efx_rx_packet_gro(channel
, rx_buf
, eh
);
620 efx_rx_deliver(channel
, rx_buf
);
623 void efx_rx_strategy(struct efx_channel
*channel
)
625 enum efx_rx_alloc_method method
= rx_alloc_method
;
627 /* Only makes sense to use page based allocation if GRO is enabled */
628 if (!(channel
->efx
->net_dev
->features
& NETIF_F_GRO
)) {
629 method
= RX_ALLOC_METHOD_SKB
;
630 } else if (method
== RX_ALLOC_METHOD_AUTO
) {
631 /* Constrain the rx_alloc_level */
632 if (channel
->rx_alloc_level
< 0)
633 channel
->rx_alloc_level
= 0;
634 else if (channel
->rx_alloc_level
> RX_ALLOC_LEVEL_MAX
)
635 channel
->rx_alloc_level
= RX_ALLOC_LEVEL_MAX
;
637 /* Decide on the allocation method */
638 method
= ((channel
->rx_alloc_level
> RX_ALLOC_LEVEL_GRO
) ?
639 RX_ALLOC_METHOD_PAGE
: RX_ALLOC_METHOD_SKB
);
642 /* Push the option */
643 channel
->rx_alloc_push_pages
= (method
== RX_ALLOC_METHOD_PAGE
);
646 int efx_probe_rx_queue(struct efx_rx_queue
*rx_queue
)
648 struct efx_nic
*efx
= rx_queue
->efx
;
649 unsigned int entries
;
652 /* Create the smallest power-of-two aligned ring */
653 entries
= max(roundup_pow_of_two(efx
->rxq_entries
), EFX_MIN_DMAQ_SIZE
);
654 EFX_BUG_ON_PARANOID(entries
> EFX_MAX_DMAQ_SIZE
);
655 rx_queue
->ptr_mask
= entries
- 1;
657 netif_dbg(efx
, probe
, efx
->net_dev
,
658 "creating RX queue %d size %#x mask %#x\n",
659 efx_rx_queue_index(rx_queue
), efx
->rxq_entries
,
662 /* Allocate RX buffers */
663 rx_queue
->buffer
= kcalloc(entries
, sizeof(*rx_queue
->buffer
),
665 if (!rx_queue
->buffer
)
668 rc
= efx_nic_probe_rx(rx_queue
);
670 kfree(rx_queue
->buffer
);
671 rx_queue
->buffer
= NULL
;
676 void efx_init_rx_queue(struct efx_rx_queue
*rx_queue
)
678 struct efx_nic
*efx
= rx_queue
->efx
;
679 unsigned int max_fill
, trigger
, max_trigger
;
681 netif_dbg(rx_queue
->efx
, drv
, rx_queue
->efx
->net_dev
,
682 "initialising RX queue %d\n", efx_rx_queue_index(rx_queue
));
684 /* Initialise ptr fields */
685 rx_queue
->added_count
= 0;
686 rx_queue
->notified_count
= 0;
687 rx_queue
->removed_count
= 0;
688 rx_queue
->min_fill
= -1U;
690 /* Initialise limit fields */
691 max_fill
= efx
->rxq_entries
- EFX_RXD_HEAD_ROOM
;
692 max_trigger
= max_fill
- EFX_RX_BATCH
;
693 if (rx_refill_threshold
!= 0) {
694 trigger
= max_fill
* min(rx_refill_threshold
, 100U) / 100U;
695 if (trigger
> max_trigger
)
696 trigger
= max_trigger
;
698 trigger
= max_trigger
;
701 rx_queue
->max_fill
= max_fill
;
702 rx_queue
->fast_fill_trigger
= trigger
;
704 /* Set up RX descriptor ring */
705 rx_queue
->enabled
= true;
706 efx_nic_init_rx(rx_queue
);
709 void efx_fini_rx_queue(struct efx_rx_queue
*rx_queue
)
712 struct efx_rx_buffer
*rx_buf
;
714 netif_dbg(rx_queue
->efx
, drv
, rx_queue
->efx
->net_dev
,
715 "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue
));
717 /* A flush failure might have left rx_queue->enabled */
718 rx_queue
->enabled
= false;
720 del_timer_sync(&rx_queue
->slow_fill
);
721 efx_nic_fini_rx(rx_queue
);
723 /* Release RX buffers NB start at index 0 not current HW ptr */
724 if (rx_queue
->buffer
) {
725 for (i
= 0; i
<= rx_queue
->ptr_mask
; i
++) {
726 rx_buf
= efx_rx_buffer(rx_queue
, i
);
727 efx_fini_rx_buffer(rx_queue
, rx_buf
);
732 void efx_remove_rx_queue(struct efx_rx_queue
*rx_queue
)
734 netif_dbg(rx_queue
->efx
, drv
, rx_queue
->efx
->net_dev
,
735 "destroying RX queue %d\n", efx_rx_queue_index(rx_queue
));
737 efx_nic_remove_rx(rx_queue
);
739 kfree(rx_queue
->buffer
);
740 rx_queue
->buffer
= NULL
;
744 module_param(rx_alloc_method
, int, 0644);
745 MODULE_PARM_DESC(rx_alloc_method
, "Allocation method used for RX buffers");
747 module_param(rx_refill_threshold
, uint
, 0444);
748 MODULE_PARM_DESC(rx_refill_threshold
,
749 "RX descriptor ring refill threshold (%)");