projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sfc: Set net_device::vlan_features appropriately
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / net / sfc / tx.c
CommitLineData
8ceee660
BH		 1/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2008 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/pci.h>
12#include <linux/tcp.h>
13#include <linux/ip.h>
14#include <linux/in.h>
15#include <linux/if_ether.h>
16#include <linux/highmem.h>
17#include "net_driver.h"
18#include "tx.h"
19#include "efx.h"
20#include "falcon.h"
21#include "workarounds.h"
22
23/*
24 * TX descriptor ring full threshold
25 *
26 * The tx_queue descriptor ring fill-level must fall below this value
27 * before we restart the netif queue
28 */
29#define EFX_NETDEV_TX_THRESHOLD(_tx_queue) \
30 (_tx_queue->efx->type->txd_ring_mask / 2u)
31
32/* We want to be able to nest calls to netif_stop_queue(), since each
33 * channel can have an individual stop on the queue.
34 */
35void efx_stop_queue(struct efx_nic *efx)
36{
37 spin_lock_bh(&efx->netif_stop_lock);
38 EFX_TRACE(efx, "stop TX queue\n");
39
40 atomic_inc(&efx->netif_stop_count);
41 netif_stop_queue(efx->net_dev);
42
43 spin_unlock_bh(&efx->netif_stop_lock);
44}
45
46/* Wake netif's TX queue
47 * We want to be able to nest calls to netif_stop_queue(), since each
48 * channel can have an individual stop on the queue.
49 */
50inline void efx_wake_queue(struct efx_nic *efx)
51{
52 local_bh_disable();
53 if (atomic_dec_and_lock(&efx->netif_stop_count,
54 &efx->netif_stop_lock)) {
55 EFX_TRACE(efx, "waking TX queue\n");
56 netif_wake_queue(efx->net_dev);
57 spin_unlock(&efx->netif_stop_lock);
58 }
59 local_bh_enable();
60}
61
62static inline void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
63 struct efx_tx_buffer *buffer)
64{
65 if (buffer->unmap_len) {
66 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
cc12dac2
BH		 67 dma_addr_t unmap_addr = (buffer->dma_addr + buffer->len -
68 buffer->unmap_len);
8ceee660 69 if (buffer->unmap_single)
cc12dac2
BH		 70 pci_unmap_single(pci_dev, unmap_addr, buffer->unmap_len,
71 PCI_DMA_TODEVICE);
8ceee660 72 else
cc12dac2
BH		 73 pci_unmap_page(pci_dev, unmap_addr, buffer->unmap_len,
74 PCI_DMA_TODEVICE);
8ceee660 75 buffer->unmap_len = 0;
dc8cfa55 76 buffer->unmap_single = false;
8ceee660
BH		 77 }
78
79 if (buffer->skb) {
80 dev_kfree_skb_any((struct sk_buff *) buffer->skb);
81 buffer->skb = NULL;
82 EFX_TRACE(tx_queue->efx, "TX queue %d transmission id %x "
83 "complete\n", tx_queue->queue, read_ptr);
84 }
85}
86
b9b39b62
BH		 87/**
88 * struct efx_tso_header - a DMA mapped buffer for packet headers
89 * @next: Linked list of free ones.
90 * The list is protected by the TX queue lock.
91 * @dma_unmap_len: Length to unmap for an oversize buffer, or 0.
92 * @dma_addr: The DMA address of the header below.
93 *
94 * This controls the memory used for a TSO header. Use TSOH_DATA()
95 * to find the packet header data. Use TSOH_SIZE() to calculate the
96 * total size required for a given packet header length. TSO headers
97 * in the free list are exactly %TSOH_STD_SIZE bytes in size.
98 */
99struct efx_tso_header {
100 union {
101 struct efx_tso_header *next;
102 size_t unmap_len;
103 };
104 dma_addr_t dma_addr;
105};
106
107static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
108 const struct sk_buff *skb);
109static void efx_fini_tso(struct efx_tx_queue *tx_queue);
110static void efx_tsoh_heap_free(struct efx_tx_queue *tx_queue,
111 struct efx_tso_header *tsoh);
112
113static inline void efx_tsoh_free(struct efx_tx_queue *tx_queue,
114 struct efx_tx_buffer *buffer)
115{
116 if (buffer->tsoh) {
117 if (likely(!buffer->tsoh->unmap_len)) {
118 buffer->tsoh->next = tx_queue->tso_headers_free;
119 tx_queue->tso_headers_free = buffer->tsoh;
120 } else {
121 efx_tsoh_heap_free(tx_queue, buffer->tsoh);
122 }
123 buffer->tsoh = NULL;
124 }
125}
126
8ceee660
BH		 127
128/*
129 * Add a socket buffer to a TX queue
130 *
131 * This maps all fragments of a socket buffer for DMA and adds them to
132 * the TX queue. The queue's insert pointer will be incremented by
133 * the number of fragments in the socket buffer.
134 *
135 * If any DMA mapping fails, any mapped fragments will be unmapped,
136 * the queue's insert pointer will be restored to its original value.
137 *
138 * Returns NETDEV_TX_OK or NETDEV_TX_BUSY
139 * You must hold netif_tx_lock() to call this function.
140 */
141static inline int efx_enqueue_skb(struct efx_tx_queue *tx_queue,
142 const struct sk_buff *skb)
143{
144 struct efx_nic *efx = tx_queue->efx;
145 struct pci_dev *pci_dev = efx->pci_dev;
146 struct efx_tx_buffer *buffer;
147 skb_frag_t *fragment;
148 struct page *page;
149 int page_offset;
150 unsigned int len, unmap_len = 0, fill_level, insert_ptr, misalign;
151 dma_addr_t dma_addr, unmap_addr = 0;
152 unsigned int dma_len;
dc8cfa55 153 bool unmap_single;
8ceee660
BH		 154 int q_space, i = 0;
155 int rc = NETDEV_TX_OK;
156
157 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
158
b9b39b62
BH		 159 if (skb_shinfo((struct sk_buff *)skb)->gso_size)
160 return efx_enqueue_skb_tso(tx_queue, skb);
161
8ceee660
BH		 162 /* Get size of the initial fragment */
163 len = skb_headlen(skb);
164
165 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
166 q_space = efx->type->txd_ring_mask - 1 - fill_level;
167
168 /* Map for DMA. Use pci_map_single rather than pci_map_page
169 * since this is more efficient on machines with sparse
170 * memory.
171 */
dc8cfa55 172 unmap_single = true;
8ceee660
BH		 173 dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
174
175 /* Process all fragments */
176 while (1) {
8d8bb39b 177 if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
8ceee660
BH		 178 goto pci_err;
179
180 /* Store fields for marking in the per-fragment final
181 * descriptor */
182 unmap_len = len;
183 unmap_addr = dma_addr;
184
185 /* Add to TX queue, splitting across DMA boundaries */
186 do {
187 if (unlikely(q_space-- <= 0)) {
188 /* It might be that completions have
189 * happened since the xmit path last
190 * checked. Update the xmit path's
191 * copy of read_count.
192 */
193 ++tx_queue->stopped;
194 /* This memory barrier protects the
195 * change of stopped from the access
196 * of read_count. */
197 smp_mb();
198 tx_queue->old_read_count =
199 *(volatile unsigned *)
200 &tx_queue->read_count;
201 fill_level = (tx_queue->insert_count
202 - tx_queue->old_read_count);
203 q_space = (efx->type->txd_ring_mask - 1 -
204 fill_level);
205 if (unlikely(q_space-- <= 0))
206 goto stop;
207 smp_mb();
208 --tx_queue->stopped;
209 }
210
211 insert_ptr = (tx_queue->insert_count &
212 efx->type->txd_ring_mask);
213 buffer = &tx_queue->buffer[insert_ptr];
b9b39b62
BH		 214 efx_tsoh_free(tx_queue, buffer);
215 EFX_BUG_ON_PARANOID(buffer->tsoh);
8ceee660
BH		 216 EFX_BUG_ON_PARANOID(buffer->skb);
217 EFX_BUG_ON_PARANOID(buffer->len);
dc8cfa55 218 EFX_BUG_ON_PARANOID(!buffer->continuation);
8ceee660
BH		 219 EFX_BUG_ON_PARANOID(buffer->unmap_len);
220
221 dma_len = (((~dma_addr) & efx->type->tx_dma_mask) + 1);
222 if (likely(dma_len > len))
223 dma_len = len;
224
225 misalign = (unsigned)dma_addr & efx->type->bug5391_mask;
226 if (misalign && dma_len + misalign > 512)
227 dma_len = 512 - misalign;
228
229 /* Fill out per descriptor fields */
230 buffer->len = dma_len;
231 buffer->dma_addr = dma_addr;
232 len -= dma_len;
233 dma_addr += dma_len;
234 ++tx_queue->insert_count;
235 } while (len);
236
237 /* Transfer ownership of the unmapping to the final buffer */
8ceee660
BH		 238 buffer->unmap_single = unmap_single;
239 buffer->unmap_len = unmap_len;
240 unmap_len = 0;
241
242 /* Get address and size of next fragment */
243 if (i >= skb_shinfo(skb)->nr_frags)
244 break;
245 fragment = &skb_shinfo(skb)->frags[i];
246 len = fragment->size;
247 page = fragment->page;
248 page_offset = fragment->page_offset;
249 i++;
250 /* Map for DMA */
dc8cfa55 251 unmap_single = false;
8ceee660
BH		 252 dma_addr = pci_map_page(pci_dev, page, page_offset, len,
253 PCI_DMA_TODEVICE);
254 }
255
256 /* Transfer ownership of the skb to the final buffer */
257 buffer->skb = skb;
dc8cfa55 258 buffer->continuation = false;
8ceee660
BH		 259
260 /* Pass off to hardware */
261 falcon_push_buffers(tx_queue);
262
263 return NETDEV_TX_OK;
264
265 pci_err:
266 EFX_ERR_RL(efx, " TX queue %d could not map skb with %d bytes %d "
267 "fragments for DMA\n", tx_queue->queue, skb->len,
268 skb_shinfo(skb)->nr_frags + 1);
269
270 /* Mark the packet as transmitted, and free the SKB ourselves */
271 dev_kfree_skb_any((struct sk_buff *)skb);
272 goto unwind;
273
274 stop:
275 rc = NETDEV_TX_BUSY;
276
277 if (tx_queue->stopped == 1)
278 efx_stop_queue(efx);
279
280 unwind:
281 /* Work backwards until we hit the original insert pointer value */
282 while (tx_queue->insert_count != tx_queue->write_count) {
283 --tx_queue->insert_count;
284 insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask;
285 buffer = &tx_queue->buffer[insert_ptr];
286 efx_dequeue_buffer(tx_queue, buffer);
287 buffer->len = 0;
288 }
289
290 /* Free the fragment we were mid-way through pushing */
ecbd95c1
BH		 291 if (unmap_len) {
292 if (unmap_single)
293 pci_unmap_single(pci_dev, unmap_addr, unmap_len,
294 PCI_DMA_TODEVICE);
295 else
296 pci_unmap_page(pci_dev, unmap_addr, unmap_len,
297 PCI_DMA_TODEVICE);
298 }
8ceee660
BH		 299
300 return rc;
301}
302
303/* Remove packets from the TX queue
304 *
305 * This removes packets from the TX queue, up to and including the
306 * specified index.
307 */
308static inline void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
309 unsigned int index)
310{
311 struct efx_nic *efx = tx_queue->efx;
312 unsigned int stop_index, read_ptr;
313 unsigned int mask = tx_queue->efx->type->txd_ring_mask;
314
315 stop_index = (index + 1) & mask;
316 read_ptr = tx_queue->read_count & mask;
317
318 while (read_ptr != stop_index) {
319 struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
320 if (unlikely(buffer->len == 0)) {
321 EFX_ERR(tx_queue->efx, "TX queue %d spurious TX "
322 "completion id %x\n", tx_queue->queue,
323 read_ptr);
324 efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
325 return;
326 }
327
328 efx_dequeue_buffer(tx_queue, buffer);
dc8cfa55 329 buffer->continuation = true;
8ceee660
BH		 330 buffer->len = 0;
331
332 ++tx_queue->read_count;
333 read_ptr = tx_queue->read_count & mask;
334 }
335}
336
337/* Initiate a packet transmission on the specified TX queue.
338 * Note that returning anything other than NETDEV_TX_OK will cause the
339 * OS to free the skb.
340 *
341 * This function is split out from efx_hard_start_xmit to allow the
342 * loopback test to direct packets via specific TX queues. It is
343 * therefore a non-static inline, so as not to penalise performance
344 * for non-loopback transmissions.
345 *
346 * Context: netif_tx_lock held
347 */
348inline int efx_xmit(struct efx_nic *efx,
349 struct efx_tx_queue *tx_queue, struct sk_buff *skb)
350{
351 int rc;
352
353 /* Map fragments for DMA and add to TX queue */
354 rc = efx_enqueue_skb(tx_queue, skb);
355 if (unlikely(rc != NETDEV_TX_OK))
356 goto out;
357
358 /* Update last TX timer */
359 efx->net_dev->trans_start = jiffies;
360
361 out:
362 return rc;
363}
364
365/* Initiate a packet transmission. We use one channel per CPU
366 * (sharing when we have more CPUs than channels). On Falcon, the TX
367 * completion events will be directed back to the CPU that transmitted
368 * the packet, which should be cache-efficient.
369 *
370 * Context: non-blocking.
371 * Note that returning anything other than NETDEV_TX_OK will cause the
372 * OS to free the skb.
373 */
374int efx_hard_start_xmit(struct sk_buff *skb, struct net_device *net_dev)
375{
767e468c 376 struct efx_nic *efx = netdev_priv(net_dev);
60ac1065
BH		 377 struct efx_tx_queue *tx_queue;
378
379 if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
380 tx_queue = &efx->tx_queue[EFX_TX_QUEUE_OFFLOAD_CSUM];
381 else
382 tx_queue = &efx->tx_queue[EFX_TX_QUEUE_NO_CSUM];
383
384 return efx_xmit(efx, tx_queue, skb);
8ceee660
BH		 385}
386
387void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
388{
389 unsigned fill_level;
390 struct efx_nic *efx = tx_queue->efx;
391
392 EFX_BUG_ON_PARANOID(index > efx->type->txd_ring_mask);
393
394 efx_dequeue_buffers(tx_queue, index);
395
396 /* See if we need to restart the netif queue. This barrier
397 * separates the update of read_count from the test of
398 * stopped. */
399 smp_mb();
400 if (unlikely(tx_queue->stopped)) {
401 fill_level = tx_queue->insert_count - tx_queue->read_count;
402 if (fill_level < EFX_NETDEV_TX_THRESHOLD(tx_queue)) {
55668611 403 EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
8ceee660
BH		 404
405 /* Do this under netif_tx_lock(), to avoid racing
406 * with efx_xmit(). */
407 netif_tx_lock(efx->net_dev);
408 if (tx_queue->stopped) {
409 tx_queue->stopped = 0;
410 efx_wake_queue(efx);
411 }
412 netif_tx_unlock(efx->net_dev);
413 }
414 }
415}
416
417int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
418{
419 struct efx_nic *efx = tx_queue->efx;
420 unsigned int txq_size;
421 int i, rc;
422
423 EFX_LOG(efx, "creating TX queue %d\n", tx_queue->queue);
424
425 /* Allocate software ring */
426 txq_size = (efx->type->txd_ring_mask + 1) * sizeof(*tx_queue->buffer);
427 tx_queue->buffer = kzalloc(txq_size, GFP_KERNEL);
60ac1065
BH		 428 if (!tx_queue->buffer)
429 return -ENOMEM;
8ceee660 430 for (i = 0; i <= efx->type->txd_ring_mask; ++i)
dc8cfa55 431 tx_queue->buffer[i].continuation = true;
8ceee660
BH		 432
433 /* Allocate hardware ring */
434 rc = falcon_probe_tx(tx_queue);
435 if (rc)
60ac1065 436 goto fail;
8ceee660
BH		 437
438 return 0;
439
60ac1065 440 fail:
8ceee660
BH		 441 kfree(tx_queue->buffer);
442 tx_queue->buffer = NULL;
8ceee660
BH		 443 return rc;
444}
445
446int efx_init_tx_queue(struct efx_tx_queue *tx_queue)
447{
448 EFX_LOG(tx_queue->efx, "initialising TX queue %d\n", tx_queue->queue);
449
450 tx_queue->insert_count = 0;
451 tx_queue->write_count = 0;
452 tx_queue->read_count = 0;
453 tx_queue->old_read_count = 0;
454 BUG_ON(tx_queue->stopped);
455
456 /* Set up TX descriptor ring */
457 return falcon_init_tx(tx_queue);
458}
459
460void efx_release_tx_buffers(struct efx_tx_queue *tx_queue)
461{
462 struct efx_tx_buffer *buffer;
463
464 if (!tx_queue->buffer)
465 return;
466
467 /* Free any buffers left in the ring */
468 while (tx_queue->read_count != tx_queue->write_count) {
469 buffer = &tx_queue->buffer[tx_queue->read_count &
470 tx_queue->efx->type->txd_ring_mask];
471 efx_dequeue_buffer(tx_queue, buffer);
dc8cfa55 472 buffer->continuation = true;
8ceee660
BH		 473 buffer->len = 0;
474
475 ++tx_queue->read_count;
476 }
477}
478
479void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
480{
481 EFX_LOG(tx_queue->efx, "shutting down TX queue %d\n", tx_queue->queue);
482
483 /* Flush TX queue, remove descriptor ring */
484 falcon_fini_tx(tx_queue);
485
486 efx_release_tx_buffers(tx_queue);
487
b9b39b62
BH		 488 /* Free up TSO header cache */
489 efx_fini_tso(tx_queue);
490
8ceee660
BH		 491 /* Release queue's stop on port, if any */
492 if (tx_queue->stopped) {
493 tx_queue->stopped = 0;
494 efx_wake_queue(tx_queue->efx);
495 }
496}
497
498void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
499{
500 EFX_LOG(tx_queue->efx, "destroying TX queue %d\n", tx_queue->queue);
501 falcon_remove_tx(tx_queue);
502
503 kfree(tx_queue->buffer);
504 tx_queue->buffer = NULL;
8ceee660
BH		 505}
506
507
b9b39b62
BH		 508/* Efx TCP segmentation acceleration.
509 *
510 * Why? Because by doing it here in the driver we can go significantly
511 * faster than the GSO.
512 *
513 * Requires TX checksum offload support.
514 */
515
516/* Number of bytes inserted at the start of a TSO header buffer,
517 * similar to NET_IP_ALIGN.
518 */
519#if defined(__i386__) || defined(__x86_64__)
520#define TSOH_OFFSET 0
521#else
522#define TSOH_OFFSET NET_IP_ALIGN
523#endif
524
525#define TSOH_BUFFER(tsoh) ((u8 *)(tsoh + 1) + TSOH_OFFSET)
526
527/* Total size of struct efx_tso_header, buffer and padding */
528#define TSOH_SIZE(hdr_len) \
529 (sizeof(struct efx_tso_header) + TSOH_OFFSET + hdr_len)
530
531/* Size of blocks on free list. Larger blocks must be allocated from
532 * the heap.
533 */
534#define TSOH_STD_SIZE 128
535
536#define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2))
537#define ETH_HDR_LEN(skb) (skb_network_header(skb) - (skb)->data)
538#define SKB_TCP_OFF(skb) PTR_DIFF(tcp_hdr(skb), (skb)->data)
539#define SKB_IPV4_OFF(skb) PTR_DIFF(ip_hdr(skb), (skb)->data)
540
541/**
542 * struct tso_state - TSO state for an SKB
543 * @remaining_len: Bytes of data we've yet to segment
544 * @seqnum: Current sequence number
545 * @packet_space: Remaining space in current packet
546 * @ifc: Input fragment cursor.
547 * Where we are in the current fragment of the incoming SKB. These
548 * values get updated in place when we split a fragment over
549 * multiple packets.
550 * @p: Parameters.
551 * These values are set once at the start of the TSO send and do
552 * not get changed as the routine progresses.
553 *
554 * The state used during segmentation. It is put into this data structure
555 * just to make it easy to pass into inline functions.
556 */
557struct tso_state {
558 unsigned remaining_len;
559 unsigned seqnum;
560 unsigned packet_space;
561
562 struct {
563 /* DMA address of current position */
564 dma_addr_t dma_addr;
565 /* Remaining length */
566 unsigned int len;
567 /* DMA address and length of the whole fragment */
568 unsigned int unmap_len;
569 dma_addr_t unmap_addr;
dc8cfa55 570 bool unmap_single;
b9b39b62
BH		 571 } ifc;
572
573 struct {
574 /* The number of bytes of header */
575 unsigned int header_length;
576
577 /* The number of bytes to put in each outgoing segment. */
578 int full_packet_size;
579
580 /* Current IPv4 ID, host endian. */
581 unsigned ipv4_id;
582 } p;
583};
584
585
586/*
587 * Verify that our various assumptions about sk_buffs and the conditions
588 * under which TSO will be attempted hold true.
589 */
590static inline void efx_tso_check_safe(const struct sk_buff *skb)
591{
592 EFX_BUG_ON_PARANOID(skb->protocol != htons(ETH_P_IP));
593 EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
594 skb->protocol);
595 EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
596 EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
597 + (tcp_hdr(skb)->doff << 2u)) >
598 skb_headlen(skb));
599}
600
601
602/*
603 * Allocate a page worth of efx_tso_header structures, and string them
604 * into the tx_queue->tso_headers_free linked list. Return 0 or -ENOMEM.
605 */
606static int efx_tsoh_block_alloc(struct efx_tx_queue *tx_queue)
607{
608
609 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
610 struct efx_tso_header *tsoh;
611 dma_addr_t dma_addr;
612 u8 *base_kva, *kva;
613
614 base_kva = pci_alloc_consistent(pci_dev, PAGE_SIZE, &dma_addr);
615 if (base_kva == NULL) {
616 EFX_ERR(tx_queue->efx, "Unable to allocate page for TSO"
617 " headers\n");
618 return -ENOMEM;
619 }
620
621 /* pci_alloc_consistent() allocates pages. */
622 EFX_BUG_ON_PARANOID(dma_addr & (PAGE_SIZE - 1u));
623
624 for (kva = base_kva; kva < base_kva + PAGE_SIZE; kva += TSOH_STD_SIZE) {
625 tsoh = (struct efx_tso_header *)kva;
626 tsoh->dma_addr = dma_addr + (TSOH_BUFFER(tsoh) - base_kva);
627 tsoh->next = tx_queue->tso_headers_free;
628 tx_queue->tso_headers_free = tsoh;
629 }
630
631 return 0;
632}
633
634
635/* Free up a TSO header, and all others in the same page. */
636static void efx_tsoh_block_free(struct efx_tx_queue *tx_queue,
637 struct efx_tso_header *tsoh,
638 struct pci_dev *pci_dev)
639{
640 struct efx_tso_header **p;
641 unsigned long base_kva;
642 dma_addr_t base_dma;
643
644 base_kva = (unsigned long)tsoh & PAGE_MASK;
645 base_dma = tsoh->dma_addr & PAGE_MASK;
646
647 p = &tx_queue->tso_headers_free;
b3475645 648 while (*p != NULL) {
b9b39b62
BH		 649 if (((unsigned long)*p & PAGE_MASK) == base_kva)
650 *p = (*p)->next;
651 else
652 p = &(*p)->next;
b3475645 653 }
b9b39b62
BH		 654
655 pci_free_consistent(pci_dev, PAGE_SIZE, (void *)base_kva, base_dma);
656}
657
658static struct efx_tso_header *
659efx_tsoh_heap_alloc(struct efx_tx_queue *tx_queue, size_t header_len)
660{
661 struct efx_tso_header *tsoh;
662
663 tsoh = kmalloc(TSOH_SIZE(header_len), GFP_ATOMIC | GFP_DMA);
664 if (unlikely(!tsoh))
665 return NULL;
666
667 tsoh->dma_addr = pci_map_single(tx_queue->efx->pci_dev,
668 TSOH_BUFFER(tsoh), header_len,
669 PCI_DMA_TODEVICE);
8d8bb39b
FT		 670 if (unlikely(pci_dma_mapping_error(tx_queue->efx->pci_dev,
671 tsoh->dma_addr))) {
b9b39b62
BH		 672 kfree(tsoh);
673 return NULL;
674 }
675
676 tsoh->unmap_len = header_len;
677 return tsoh;
678}
679
680static void
681efx_tsoh_heap_free(struct efx_tx_queue *tx_queue, struct efx_tso_header *tsoh)
682{
683 pci_unmap_single(tx_queue->efx->pci_dev,
684 tsoh->dma_addr, tsoh->unmap_len,
685 PCI_DMA_TODEVICE);
686 kfree(tsoh);
687}
688
689/**
690 * efx_tx_queue_insert - push descriptors onto the TX queue
691 * @tx_queue: Efx TX queue
692 * @dma_addr: DMA address of fragment
693 * @len: Length of fragment
ecbd95c1 694 * @final_buffer: The final buffer inserted into the queue
b9b39b62
BH		 695 *
696 * Push descriptors onto the TX queue. Return 0 on success or 1 if
697 * @tx_queue full.
698 */
699static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
700 dma_addr_t dma_addr, unsigned len,
ecbd95c1 701 struct efx_tx_buffer **final_buffer)
b9b39b62
BH		 702{
703 struct efx_tx_buffer *buffer;
704 struct efx_nic *efx = tx_queue->efx;
705 unsigned dma_len, fill_level, insert_ptr, misalign;
706 int q_space;
707
708 EFX_BUG_ON_PARANOID(len <= 0);
709
710 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
711 /* -1 as there is no way to represent all descriptors used */
712 q_space = efx->type->txd_ring_mask - 1 - fill_level;
713
714 while (1) {
715 if (unlikely(q_space-- <= 0)) {
716 /* It might be that completions have happened
717 * since the xmit path last checked. Update
718 * the xmit path's copy of read_count.
719 */
720 ++tx_queue->stopped;
721 /* This memory barrier protects the change of
722 * stopped from the access of read_count. */
723 smp_mb();
724 tx_queue->old_read_count =
725 *(volatile unsigned *)&tx_queue->read_count;
726 fill_level = (tx_queue->insert_count
727 - tx_queue->old_read_count);
728 q_space = efx->type->txd_ring_mask - 1 - fill_level;
ecbd95c1
BH		 729 if (unlikely(q_space-- <= 0)) {
730 *final_buffer = NULL;
b9b39b62 731 return 1;
ecbd95c1 732 }
b9b39b62
BH		 733 smp_mb();
734 --tx_queue->stopped;
735 }
736
737 insert_ptr = tx_queue->insert_count & efx->type->txd_ring_mask;
738 buffer = &tx_queue->buffer[insert_ptr];
739 ++tx_queue->insert_count;
740
741 EFX_BUG_ON_PARANOID(tx_queue->insert_count -
742 tx_queue->read_count >
743 efx->type->txd_ring_mask);
744
745 efx_tsoh_free(tx_queue, buffer);
746 EFX_BUG_ON_PARANOID(buffer->len);
747 EFX_BUG_ON_PARANOID(buffer->unmap_len);
748 EFX_BUG_ON_PARANOID(buffer->skb);
dc8cfa55 749 EFX_BUG_ON_PARANOID(!buffer->continuation);
b9b39b62
BH		 750 EFX_BUG_ON_PARANOID(buffer->tsoh);
751
752 buffer->dma_addr = dma_addr;
753
754 /* Ensure we do not cross a boundary unsupported by H/W */
755 dma_len = (~dma_addr & efx->type->tx_dma_mask) + 1;
756
757 misalign = (unsigned)dma_addr & efx->type->bug5391_mask;
758 if (misalign && dma_len + misalign > 512)
759 dma_len = 512 - misalign;
760
761 /* If there is enough space to send then do so */
762 if (dma_len >= len)
763 break;
764
765 buffer->len = dma_len; /* Don't set the other members */
766 dma_addr += dma_len;
767 len -= dma_len;
768 }
769
770 EFX_BUG_ON_PARANOID(!len);
771 buffer->len = len;
ecbd95c1 772 *final_buffer = buffer;
b9b39b62
BH		 773 return 0;
774}
775
776
777/*
778 * Put a TSO header into the TX queue.
779 *
780 * This is special-cased because we know that it is small enough to fit in
781 * a single fragment, and we know it doesn't cross a page boundary. It
782 * also allows us to not worry about end-of-packet etc.
783 */
784static inline void efx_tso_put_header(struct efx_tx_queue *tx_queue,
785 struct efx_tso_header *tsoh, unsigned len)
786{
787 struct efx_tx_buffer *buffer;
788
789 buffer = &tx_queue->buffer[tx_queue->insert_count &
790 tx_queue->efx->type->txd_ring_mask];
791 efx_tsoh_free(tx_queue, buffer);
792 EFX_BUG_ON_PARANOID(buffer->len);
793 EFX_BUG_ON_PARANOID(buffer->unmap_len);
794 EFX_BUG_ON_PARANOID(buffer->skb);
dc8cfa55 795 EFX_BUG_ON_PARANOID(!buffer->continuation);
b9b39b62
BH		 796 EFX_BUG_ON_PARANOID(buffer->tsoh);
797 buffer->len = len;
798 buffer->dma_addr = tsoh->dma_addr;
799 buffer->tsoh = tsoh;
800
801 ++tx_queue->insert_count;
802}
803
804
805/* Remove descriptors put into a tx_queue. */
806static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
807{
808 struct efx_tx_buffer *buffer;
cc12dac2 809 dma_addr_t unmap_addr;
b9b39b62
BH		 810
811 /* Work backwards until we hit the original insert pointer value */
812 while (tx_queue->insert_count != tx_queue->write_count) {
813 --tx_queue->insert_count;
814 buffer = &tx_queue->buffer[tx_queue->insert_count &
815 tx_queue->efx->type->txd_ring_mask];
816 efx_tsoh_free(tx_queue, buffer);
817 EFX_BUG_ON_PARANOID(buffer->skb);
818 buffer->len = 0;
dc8cfa55 819 buffer->continuation = true;
b9b39b62 820 if (buffer->unmap_len) {
cc12dac2
BH		 821 unmap_addr = (buffer->dma_addr + buffer->len -
822 buffer->unmap_len);
ecbd95c1
BH		 823 if (buffer->unmap_single)
824 pci_unmap_single(tx_queue->efx->pci_dev,
cc12dac2 825 unmap_addr, buffer->unmap_len,
ecbd95c1
BH		 826 PCI_DMA_TODEVICE);
827 else
828 pci_unmap_page(tx_queue->efx->pci_dev,
cc12dac2 829 unmap_addr, buffer->unmap_len,
ecbd95c1 830 PCI_DMA_TODEVICE);
b9b39b62
BH		 831 buffer->unmap_len = 0;
832 }
833 }
834}
835
836
837/* Parse the SKB header and initialise state. */
838static inline void tso_start(struct tso_state *st, const struct sk_buff *skb)
839{
840 /* All ethernet/IP/TCP headers combined size is TCP header size
841 * plus offset of TCP header relative to start of packet.
842 */
843 st->p.header_length = ((tcp_hdr(skb)->doff << 2u)
844 + PTR_DIFF(tcp_hdr(skb), skb->data));
845 st->p.full_packet_size = (st->p.header_length
846 + skb_shinfo(skb)->gso_size);
847
848 st->p.ipv4_id = ntohs(ip_hdr(skb)->id);
849 st->seqnum = ntohl(tcp_hdr(skb)->seq);
850
851 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
852 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
853 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
854
855 st->packet_space = st->p.full_packet_size;
856 st->remaining_len = skb->len - st->p.header_length;
ecbd95c1 857 st->ifc.unmap_len = 0;
dc8cfa55 858 st->ifc.unmap_single = false;
b9b39b62
BH		 859}
860
b9b39b62 861static inline int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
ecbd95c1 862 skb_frag_t *frag)
b9b39b62 863{
ecbd95c1
BH		 864 st->ifc.unmap_addr = pci_map_page(efx->pci_dev, frag->page,
865 frag->page_offset, frag->size,
866 PCI_DMA_TODEVICE);
867 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->ifc.unmap_addr))) {
dc8cfa55 868 st->ifc.unmap_single = false;
ecbd95c1
BH		 869 st->ifc.unmap_len = frag->size;
870 st->ifc.len = frag->size;
871 st->ifc.dma_addr = st->ifc.unmap_addr;
872 return 0;
873 }
874 return -ENOMEM;
875}
876
877static inline int
878tso_get_head_fragment(struct tso_state *st, struct efx_nic *efx,
879 const struct sk_buff *skb)
880{
881 int hl = st->p.header_length;
882 int len = skb_headlen(skb) - hl;
b9b39b62 883
ecbd95c1
BH		 884 st->ifc.unmap_addr = pci_map_single(efx->pci_dev, skb->data + hl,
885 len, PCI_DMA_TODEVICE);
8d8bb39b 886 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->ifc.unmap_addr))) {
dc8cfa55 887 st->ifc.unmap_single = true;
b9b39b62
BH		 888 st->ifc.unmap_len = len;
889 st->ifc.len = len;
890 st->ifc.dma_addr = st->ifc.unmap_addr;
b9b39b62
BH		 891 return 0;
892 }
893 return -ENOMEM;
894}
895
896
897/**
898 * tso_fill_packet_with_fragment - form descriptors for the current fragment
899 * @tx_queue: Efx TX queue
900 * @skb: Socket buffer
901 * @st: TSO state
902 *
903 * Form descriptors for the current fragment, until we reach the end
904 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
905 * space in @tx_queue.
906 */
907static inline int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
908 const struct sk_buff *skb,
909 struct tso_state *st)
910{
ecbd95c1 911 struct efx_tx_buffer *buffer;
b9b39b62
BH		 912 int n, end_of_packet, rc;
913
914 if (st->ifc.len == 0)
915 return 0;
916 if (st->packet_space == 0)
917 return 0;
918
919 EFX_BUG_ON_PARANOID(st->ifc.len <= 0);
920 EFX_BUG_ON_PARANOID(st->packet_space <= 0);
921
922 n = min(st->ifc.len, st->packet_space);
923
924 st->packet_space -= n;
925 st->remaining_len -= n;
926 st->ifc.len -= n;
b9b39b62 927
ecbd95c1
BH		 928 rc = efx_tx_queue_insert(tx_queue, st->ifc.dma_addr, n, &buffer);
929 if (likely(rc == 0)) {
930 if (st->remaining_len == 0)
931 /* Transfer ownership of the skb */
932 buffer->skb = skb;
b9b39b62 933
ecbd95c1
BH		 934 end_of_packet = st->remaining_len == 0 || st->packet_space == 0;
935 buffer->continuation = !end_of_packet;
b9b39b62 936
ecbd95c1
BH		 937 if (st->ifc.len == 0) {
938 /* Transfer ownership of the pci mapping */
939 buffer->unmap_len = st->ifc.unmap_len;
940 buffer->unmap_single = st->ifc.unmap_single;
941 st->ifc.unmap_len = 0;
942 }
943 }
944
945 st->ifc.dma_addr += n;
b9b39b62
BH		 946 return rc;
947}
948
949
950/**
951 * tso_start_new_packet - generate a new header and prepare for the new packet
952 * @tx_queue: Efx TX queue
953 * @skb: Socket buffer
954 * @st: TSO state
955 *
956 * Generate a new header and prepare for the new packet. Return 0 on
957 * success, or -1 if failed to alloc header.
958 */
959static inline int tso_start_new_packet(struct efx_tx_queue *tx_queue,
960 const struct sk_buff *skb,
961 struct tso_state *st)
962{
963 struct efx_tso_header *tsoh;
964 struct iphdr *tsoh_iph;
965 struct tcphdr *tsoh_th;
966 unsigned ip_length;
967 u8 *header;
968
969 /* Allocate a DMA-mapped header buffer. */
970 if (likely(TSOH_SIZE(st->p.header_length) <= TSOH_STD_SIZE)) {
b3475645 971 if (tx_queue->tso_headers_free == NULL) {
b9b39b62
BH		 972 if (efx_tsoh_block_alloc(tx_queue))
973 return -1;
b3475645 974 }
b9b39b62
BH		 975 EFX_BUG_ON_PARANOID(!tx_queue->tso_headers_free);
976 tsoh = tx_queue->tso_headers_free;
977 tx_queue->tso_headers_free = tsoh->next;
978 tsoh->unmap_len = 0;
979 } else {
980 tx_queue->tso_long_headers++;
981 tsoh = efx_tsoh_heap_alloc(tx_queue, st->p.header_length);
982 if (unlikely(!tsoh))
983 return -1;
984 }
985
986 header = TSOH_BUFFER(tsoh);
987 tsoh_th = (struct tcphdr *)(header + SKB_TCP_OFF(skb));
988 tsoh_iph = (struct iphdr *)(header + SKB_IPV4_OFF(skb));
989
990 /* Copy and update the headers. */
991 memcpy(header, skb->data, st->p.header_length);
992
993 tsoh_th->seq = htonl(st->seqnum);
994 st->seqnum += skb_shinfo(skb)->gso_size;
995 if (st->remaining_len > skb_shinfo(skb)->gso_size) {
996 /* This packet will not finish the TSO burst. */
997 ip_length = st->p.full_packet_size - ETH_HDR_LEN(skb);
998 tsoh_th->fin = 0;
999 tsoh_th->psh = 0;
1000 } else {
1001 /* This packet will be the last in the TSO burst. */
1002 ip_length = (st->p.header_length - ETH_HDR_LEN(skb)
1003 + st->remaining_len);
1004 tsoh_th->fin = tcp_hdr(skb)->fin;
1005 tsoh_th->psh = tcp_hdr(skb)->psh;
1006 }
1007 tsoh_iph->tot_len = htons(ip_length);
1008
1009 /* Linux leaves suitable gaps in the IP ID space for us to fill. */
1010 tsoh_iph->id = htons(st->p.ipv4_id);
1011 st->p.ipv4_id++;
1012
1013 st->packet_space = skb_shinfo(skb)->gso_size;
1014 ++tx_queue->tso_packets;
1015
1016 /* Form a descriptor for this header. */
1017 efx_tso_put_header(tx_queue, tsoh, st->p.header_length);
1018
1019 return 0;
1020}
1021
1022
1023/**
1024 * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
1025 * @tx_queue: Efx TX queue
1026 * @skb: Socket buffer
1027 *
1028 * Context: You must hold netif_tx_lock() to call this function.
1029 *
1030 * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
1031 * @skb was not enqueued. In all cases @skb is consumed. Return
1032 * %NETDEV_TX_OK or %NETDEV_TX_BUSY.
1033 */
1034static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
1035 const struct sk_buff *skb)
1036{
ecbd95c1 1037 struct efx_nic *efx = tx_queue->efx;
b9b39b62
BH		 1038 int frag_i, rc, rc2 = NETDEV_TX_OK;
1039 struct tso_state state;
b9b39b62
BH		 1040
1041 /* Verify TSO is safe - these checks should never fail. */
1042 efx_tso_check_safe(skb);
1043
1044 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
1045
1046 tso_start(&state, skb);
1047
1048 /* Assume that skb header area contains exactly the headers, and
1049 * all payload is in the frag list.
1050 */
1051 if (skb_headlen(skb) == state.p.header_length) {
1052 /* Grab the first payload fragment. */
1053 EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
1054 frag_i = 0;
ecbd95c1
BH		 1055 rc = tso_get_fragment(&state, efx,
1056 skb_shinfo(skb)->frags + frag_i);
b9b39b62
BH		 1057 if (rc)
1058 goto mem_err;
1059 } else {
ecbd95c1 1060 rc = tso_get_head_fragment(&state, efx, skb);
b9b39b62
BH		 1061 if (rc)
1062 goto mem_err;
1063 frag_i = -1;
1064 }
1065
1066 if (tso_start_new_packet(tx_queue, skb, &state) < 0)
1067 goto mem_err;
1068
1069 while (1) {
1070 rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
1071 if (unlikely(rc))
1072 goto stop;
1073
1074 /* Move onto the next fragment? */
1075 if (state.ifc.len == 0) {
1076 if (++frag_i >= skb_shinfo(skb)->nr_frags)
1077 /* End of payload reached. */
1078 break;
ecbd95c1
BH		 1079 rc = tso_get_fragment(&state, efx,
1080 skb_shinfo(skb)->frags + frag_i);
b9b39b62
BH		 1081 if (rc)
1082 goto mem_err;
1083 }
1084
1085 /* Start at new packet? */
1086 if (state.packet_space == 0 &&
1087 tso_start_new_packet(tx_queue, skb, &state) < 0)
1088 goto mem_err;
1089 }
1090
1091 /* Pass off to hardware */
1092 falcon_push_buffers(tx_queue);
1093
1094 tx_queue->tso_bursts++;
1095 return NETDEV_TX_OK;
1096
1097 mem_err:
ecbd95c1 1098 EFX_ERR(efx, "Out of memory for TSO headers, or PCI mapping error\n");
b9b39b62
BH		 1099 dev_kfree_skb_any((struct sk_buff *)skb);
1100 goto unwind;
1101
1102 stop:
1103 rc2 = NETDEV_TX_BUSY;
1104
1105 /* Stop the queue if it wasn't stopped before. */
1106 if (tx_queue->stopped == 1)
ecbd95c1 1107 efx_stop_queue(efx);
b9b39b62
BH		 1108
1109 unwind:
5988b63a 1110 /* Free the DMA mapping we were in the process of writing out */
ecbd95c1
BH		 1111 if (state.ifc.unmap_len) {
1112 if (state.ifc.unmap_single)
1113 pci_unmap_single(efx->pci_dev, state.ifc.unmap_addr,
1114 state.ifc.unmap_len, PCI_DMA_TODEVICE);
1115 else
1116 pci_unmap_page(efx->pci_dev, state.ifc.unmap_addr,
1117 state.ifc.unmap_len, PCI_DMA_TODEVICE);
1118 }
5988b63a 1119
b9b39b62
BH		 1120 efx_enqueue_unwind(tx_queue);
1121 return rc2;
1122}
1123
1124
1125/*
1126 * Free up all TSO datastructures associated with tx_queue. This
1127 * routine should be called only once the tx_queue is both empty and
1128 * will no longer be used.
1129 */
1130static void efx_fini_tso(struct efx_tx_queue *tx_queue)
1131{
1132 unsigned i;
1133
b3475645 1134 if (tx_queue->buffer) {
b9b39b62
BH		 1135 for (i = 0; i <= tx_queue->efx->type->txd_ring_mask; ++i)
1136 efx_tsoh_free(tx_queue, &tx_queue->buffer[i]);
b3475645 1137 }
b9b39b62
BH		 1138
1139 while (tx_queue->tso_headers_free != NULL)
1140 efx_tsoh_block_free(tx_queue, tx_queue->tso_headers_free,
1141 tx_queue->efx->pci_dev);
1142}
kernel source for telekom puls (alcatel/mediatek)