2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25 * does asynchronous data movement and checksumming operations.
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/workqueue.h>
36 #include <linux/i7300_idle.h>
39 #include "registers.h"
42 int ioat_ring_alloc_order
= 8;
43 module_param(ioat_ring_alloc_order
, int, 0644);
44 MODULE_PARM_DESC(ioat_ring_alloc_order
,
45 "ioat2+: allocate 2^n descriptors per channel"
46 " (default: 8 max: 16)");
47 static int ioat_ring_max_alloc_order
= IOAT_MAX_ORDER
;
48 module_param(ioat_ring_max_alloc_order
, int, 0644);
49 MODULE_PARM_DESC(ioat_ring_max_alloc_order
,
50 "ioat2+: upper limit for ring size (default: 16)");
52 void __ioat2_issue_pending(struct ioat2_dma_chan
*ioat
)
54 void * __iomem reg_base
= ioat
->base
.reg_base
;
57 ioat
->dmacount
+= ioat2_ring_pending(ioat
);
58 ioat
->issued
= ioat
->head
;
59 /* make descriptor updates globally visible before notifying channel */
61 writew(ioat
->dmacount
, reg_base
+ IOAT_CHAN_DMACOUNT_OFFSET
);
62 dev_dbg(to_dev(&ioat
->base
),
63 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
64 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
67 void ioat2_issue_pending(struct dma_chan
*chan
)
69 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(chan
);
71 spin_lock_bh(&ioat
->ring_lock
);
72 if (ioat
->pending
== 1)
73 __ioat2_issue_pending(ioat
);
74 spin_unlock_bh(&ioat
->ring_lock
);
78 * ioat2_update_pending - log pending descriptors
79 * @ioat: ioat2+ channel
81 * set pending to '1' unless pending is already set to '2', pending == 2
82 * indicates that submission is temporarily blocked due to an in-flight
83 * reset. If we are already above the ioat_pending_level threshold then
86 * called with ring_lock held
88 static void ioat2_update_pending(struct ioat2_dma_chan
*ioat
)
90 if (unlikely(ioat
->pending
== 2))
92 else if (ioat2_ring_pending(ioat
) > ioat_pending_level
)
93 __ioat2_issue_pending(ioat
);
98 static void __ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
100 struct ioat_ring_ent
*desc
;
101 struct ioat_dma_descriptor
*hw
;
104 if (ioat2_ring_space(ioat
) < 1) {
105 dev_err(to_dev(&ioat
->base
),
106 "Unable to start null desc - ring full\n");
110 dev_dbg(to_dev(&ioat
->base
), "%s: head: %#x tail: %#x issued: %#x\n",
111 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
112 idx
= ioat2_desc_alloc(ioat
, 1);
113 desc
= ioat2_get_ring_ent(ioat
, idx
);
118 hw
->ctl_f
.int_en
= 1;
119 hw
->ctl_f
.compl_write
= 1;
120 /* set size to non-zero value (channel returns error when size is 0) */
121 hw
->size
= NULL_DESC_BUFFER_SIZE
;
124 async_tx_ack(&desc
->txd
);
125 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
126 dump_desc_dbg(ioat
, desc
);
127 __ioat2_issue_pending(ioat
);
130 static void ioat2_start_null_desc(struct ioat2_dma_chan
*ioat
)
132 spin_lock_bh(&ioat
->ring_lock
);
133 __ioat2_start_null_desc(ioat
);
134 spin_unlock_bh(&ioat
->ring_lock
);
137 static void __cleanup(struct ioat2_dma_chan
*ioat
, unsigned long phys_complete
)
139 struct ioat_chan_common
*chan
= &ioat
->base
;
140 struct dma_async_tx_descriptor
*tx
;
141 struct ioat_ring_ent
*desc
;
142 bool seen_current
= false;
146 dev_dbg(to_dev(chan
), "%s: head: %#x tail: %#x issued: %#x\n",
147 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
);
149 active
= ioat2_ring_active(ioat
);
150 for (i
= 0; i
< active
&& !seen_current
; i
++) {
151 prefetch(ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
+ 1));
152 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
);
154 dump_desc_dbg(ioat
, desc
);
156 ioat_dma_unmap(chan
, tx
->flags
, desc
->len
, desc
->hw
);
157 chan
->completed_cookie
= tx
->cookie
;
160 tx
->callback(tx
->callback_param
);
165 if (tx
->phys
== phys_complete
)
169 BUG_ON(!seen_current
); /* no active descs have written a completion? */
171 chan
->last_completion
= phys_complete
;
172 if (ioat
->head
== ioat
->tail
) {
173 dev_dbg(to_dev(chan
), "%s: cancel completion timeout\n",
175 clear_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
176 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
181 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
182 * @chan: ioat channel to be cleaned up
184 static void ioat2_cleanup(struct ioat2_dma_chan
*ioat
)
186 struct ioat_chan_common
*chan
= &ioat
->base
;
187 unsigned long phys_complete
;
189 prefetch(chan
->completion
);
191 if (!spin_trylock_bh(&chan
->cleanup_lock
))
194 if (!ioat_cleanup_preamble(chan
, &phys_complete
)) {
195 spin_unlock_bh(&chan
->cleanup_lock
);
199 if (!spin_trylock_bh(&ioat
->ring_lock
)) {
200 spin_unlock_bh(&chan
->cleanup_lock
);
204 __cleanup(ioat
, phys_complete
);
206 spin_unlock_bh(&ioat
->ring_lock
);
207 spin_unlock_bh(&chan
->cleanup_lock
);
210 void ioat2_cleanup_tasklet(unsigned long data
)
212 struct ioat2_dma_chan
*ioat
= (void *) data
;
215 writew(IOAT_CHANCTRL_RUN
, ioat
->base
.reg_base
+ IOAT_CHANCTRL_OFFSET
);
218 void __ioat2_restart_chan(struct ioat2_dma_chan
*ioat
)
220 struct ioat_chan_common
*chan
= &ioat
->base
;
222 /* set the tail to be re-issued */
223 ioat
->issued
= ioat
->tail
;
225 set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
);
226 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
228 dev_dbg(to_dev(chan
),
229 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
230 __func__
, ioat
->head
, ioat
->tail
, ioat
->issued
, ioat
->dmacount
);
232 if (ioat2_ring_pending(ioat
)) {
233 struct ioat_ring_ent
*desc
;
235 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
);
236 ioat2_set_chainaddr(ioat
, desc
->txd
.phys
);
237 __ioat2_issue_pending(ioat
);
239 __ioat2_start_null_desc(ioat
);
242 static void ioat2_restart_channel(struct ioat2_dma_chan
*ioat
)
244 struct ioat_chan_common
*chan
= &ioat
->base
;
245 unsigned long phys_complete
;
248 status
= ioat_chansts(chan
);
249 if (is_ioat_active(status
) || is_ioat_idle(status
))
251 while (is_ioat_active(status
) || is_ioat_idle(status
)) {
252 status
= ioat_chansts(chan
);
256 if (ioat_cleanup_preamble(chan
, &phys_complete
))
257 __cleanup(ioat
, phys_complete
);
259 __ioat2_restart_chan(ioat
);
262 void ioat2_timer_event(unsigned long data
)
264 struct ioat2_dma_chan
*ioat
= (void *) data
;
265 struct ioat_chan_common
*chan
= &ioat
->base
;
267 spin_lock_bh(&chan
->cleanup_lock
);
268 if (test_bit(IOAT_COMPLETION_PENDING
, &chan
->state
)) {
269 unsigned long phys_complete
;
272 spin_lock_bh(&ioat
->ring_lock
);
273 status
= ioat_chansts(chan
);
275 /* when halted due to errors check for channel
276 * programming errors before advancing the completion state
278 if (is_ioat_halted(status
)) {
281 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
282 dev_err(to_dev(chan
), "%s: Channel halted (%x)\n",
284 BUG_ON(is_ioat_bug(chanerr
));
287 /* if we haven't made progress and we have already
288 * acknowledged a pending completion once, then be more
289 * forceful with a restart
291 if (ioat_cleanup_preamble(chan
, &phys_complete
))
292 __cleanup(ioat
, phys_complete
);
293 else if (test_bit(IOAT_COMPLETION_ACK
, &chan
->state
))
294 ioat2_restart_channel(ioat
);
296 set_bit(IOAT_COMPLETION_ACK
, &chan
->state
);
297 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
299 spin_unlock_bh(&ioat
->ring_lock
);
303 /* if the ring is idle, empty, and oversized try to step
306 spin_lock_bh(&ioat
->ring_lock
);
307 active
= ioat2_ring_active(ioat
);
308 if (active
== 0 && ioat
->alloc_order
> ioat_get_alloc_order())
309 reshape_ring(ioat
, ioat
->alloc_order
-1);
310 spin_unlock_bh(&ioat
->ring_lock
);
312 /* keep shrinking until we get back to our minimum
315 if (ioat
->alloc_order
> ioat_get_alloc_order())
316 mod_timer(&chan
->timer
, jiffies
+ IDLE_TIMEOUT
);
318 spin_unlock_bh(&chan
->cleanup_lock
);
322 * ioat2_enumerate_channels - find and initialize the device's channels
323 * @device: the device to be enumerated
325 int ioat2_enumerate_channels(struct ioatdma_device
*device
)
327 struct ioat2_dma_chan
*ioat
;
328 struct device
*dev
= &device
->pdev
->dev
;
329 struct dma_device
*dma
= &device
->common
;
333 INIT_LIST_HEAD(&dma
->channels
);
334 dma
->chancnt
= readb(device
->reg_base
+ IOAT_CHANCNT_OFFSET
);
335 dma
->chancnt
&= 0x1f; /* bits [4:0] valid */
336 if (dma
->chancnt
> ARRAY_SIZE(device
->idx
)) {
337 dev_warn(dev
, "(%d) exceeds max supported channels (%zu)\n",
338 dma
->chancnt
, ARRAY_SIZE(device
->idx
));
339 dma
->chancnt
= ARRAY_SIZE(device
->idx
);
341 xfercap_log
= readb(device
->reg_base
+ IOAT_XFERCAP_OFFSET
);
342 xfercap_log
&= 0x1f; /* bits [4:0] valid */
343 if (xfercap_log
== 0)
345 dev_dbg(dev
, "%s: xfercap = %d\n", __func__
, 1 << xfercap_log
);
347 /* FIXME which i/oat version is i7300? */
348 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
349 if (i7300_idle_platform_probe(NULL
, NULL
, 1) == 0)
352 for (i
= 0; i
< dma
->chancnt
; i
++) {
353 ioat
= devm_kzalloc(dev
, sizeof(*ioat
), GFP_KERNEL
);
357 ioat_init_channel(device
, &ioat
->base
, i
,
359 device
->cleanup_tasklet
,
360 (unsigned long) ioat
);
361 ioat
->xfercap_log
= xfercap_log
;
362 spin_lock_init(&ioat
->ring_lock
);
368 static dma_cookie_t
ioat2_tx_submit_unlock(struct dma_async_tx_descriptor
*tx
)
370 struct dma_chan
*c
= tx
->chan
;
371 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
372 struct ioat_chan_common
*chan
= &ioat
->base
;
373 dma_cookie_t cookie
= c
->cookie
;
380 dev_dbg(to_dev(&ioat
->base
), "%s: cookie: %d\n", __func__
, cookie
);
382 if (!test_and_set_bit(IOAT_COMPLETION_PENDING
, &chan
->state
))
383 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
384 ioat2_update_pending(ioat
);
385 spin_unlock_bh(&ioat
->ring_lock
);
390 static struct ioat_ring_ent
*ioat2_alloc_ring_ent(struct dma_chan
*chan
, gfp_t flags
)
392 struct ioat_dma_descriptor
*hw
;
393 struct ioat_ring_ent
*desc
;
394 struct ioatdma_device
*dma
;
397 dma
= to_ioatdma_device(chan
->device
);
398 hw
= pci_pool_alloc(dma
->dma_pool
, flags
, &phys
);
401 memset(hw
, 0, sizeof(*hw
));
403 desc
= kmem_cache_alloc(ioat2_cache
, flags
);
405 pci_pool_free(dma
->dma_pool
, hw
, phys
);
408 memset(desc
, 0, sizeof(*desc
));
410 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
411 desc
->txd
.tx_submit
= ioat2_tx_submit_unlock
;
413 desc
->txd
.phys
= phys
;
417 static void ioat2_free_ring_ent(struct ioat_ring_ent
*desc
, struct dma_chan
*chan
)
419 struct ioatdma_device
*dma
;
421 dma
= to_ioatdma_device(chan
->device
);
422 pci_pool_free(dma
->dma_pool
, desc
->hw
, desc
->txd
.phys
);
423 kmem_cache_free(ioat2_cache
, desc
);
426 static struct ioat_ring_ent
**ioat2_alloc_ring(struct dma_chan
*c
, int order
, gfp_t flags
)
428 struct ioat_ring_ent
**ring
;
429 int descs
= 1 << order
;
432 if (order
> ioat_get_max_alloc_order())
435 /* allocate the array to hold the software ring */
436 ring
= kcalloc(descs
, sizeof(*ring
), flags
);
439 for (i
= 0; i
< descs
; i
++) {
440 ring
[i
] = ioat2_alloc_ring_ent(c
, flags
);
443 ioat2_free_ring_ent(ring
[i
], c
);
447 set_desc_id(ring
[i
], i
);
451 for (i
= 0; i
< descs
-1; i
++) {
452 struct ioat_ring_ent
*next
= ring
[i
+1];
453 struct ioat_dma_descriptor
*hw
= ring
[i
]->hw
;
455 hw
->next
= next
->txd
.phys
;
457 ring
[i
]->hw
->next
= ring
[0]->txd
.phys
;
462 /* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
463 * @chan: channel to be initialized
465 int ioat2_alloc_chan_resources(struct dma_chan
*c
)
467 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
468 struct ioat_chan_common
*chan
= &ioat
->base
;
469 struct ioat_ring_ent
**ring
;
473 /* have we already been set up? */
475 return 1 << ioat
->alloc_order
;
477 /* Setup register to interrupt and write completion status on error */
478 writew(IOAT_CHANCTRL_RUN
, chan
->reg_base
+ IOAT_CHANCTRL_OFFSET
);
480 chanerr
= readl(chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
482 dev_err(to_dev(chan
), "CHANERR = %x, clearing\n", chanerr
);
483 writel(chanerr
, chan
->reg_base
+ IOAT_CHANERR_OFFSET
);
486 /* allocate a completion writeback area */
487 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
488 chan
->completion
= pci_pool_alloc(chan
->device
->completion_pool
,
489 GFP_KERNEL
, &chan
->completion_dma
);
490 if (!chan
->completion
)
493 memset(chan
->completion
, 0, sizeof(*chan
->completion
));
494 writel(((u64
) chan
->completion_dma
) & 0x00000000FFFFFFFF,
495 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_LOW
);
496 writel(((u64
) chan
->completion_dma
) >> 32,
497 chan
->reg_base
+ IOAT_CHANCMP_OFFSET_HIGH
);
499 order
= ioat_get_alloc_order();
500 ring
= ioat2_alloc_ring(c
, order
, GFP_KERNEL
);
504 spin_lock_bh(&ioat
->ring_lock
);
510 ioat
->alloc_order
= order
;
511 spin_unlock_bh(&ioat
->ring_lock
);
513 tasklet_enable(&chan
->cleanup_task
);
514 ioat2_start_null_desc(ioat
);
516 return 1 << ioat
->alloc_order
;
519 bool reshape_ring(struct ioat2_dma_chan
*ioat
, int order
)
521 /* reshape differs from normal ring allocation in that we want
522 * to allocate a new software ring while only
523 * extending/truncating the hardware ring
525 struct ioat_chan_common
*chan
= &ioat
->base
;
526 struct dma_chan
*c
= &chan
->common
;
527 const u16 curr_size
= ioat2_ring_mask(ioat
) + 1;
528 const u16 active
= ioat2_ring_active(ioat
);
529 const u16 new_size
= 1 << order
;
530 struct ioat_ring_ent
**ring
;
533 if (order
> ioat_get_max_alloc_order())
536 /* double check that we have at least 1 free descriptor */
537 if (active
== curr_size
)
540 /* when shrinking, verify that we can hold the current active
541 * set in the new ring
543 if (active
>= new_size
)
546 /* allocate the array to hold the software ring */
547 ring
= kcalloc(new_size
, sizeof(*ring
), GFP_NOWAIT
);
551 /* allocate/trim descriptors as needed */
552 if (new_size
> curr_size
) {
553 /* copy current descriptors to the new ring */
554 for (i
= 0; i
< curr_size
; i
++) {
555 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
556 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
558 ring
[new_idx
] = ioat
->ring
[curr_idx
];
559 set_desc_id(ring
[new_idx
], new_idx
);
562 /* add new descriptors to the ring */
563 for (i
= curr_size
; i
< new_size
; i
++) {
564 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
566 ring
[new_idx
] = ioat2_alloc_ring_ent(c
, GFP_NOWAIT
);
567 if (!ring
[new_idx
]) {
569 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
571 ioat2_free_ring_ent(ring
[new_idx
], c
);
576 set_desc_id(ring
[new_idx
], new_idx
);
579 /* hw link new descriptors */
580 for (i
= curr_size
-1; i
< new_size
; i
++) {
581 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
582 struct ioat_ring_ent
*next
= ring
[(new_idx
+1) & (new_size
-1)];
583 struct ioat_dma_descriptor
*hw
= ring
[new_idx
]->hw
;
585 hw
->next
= next
->txd
.phys
;
588 struct ioat_dma_descriptor
*hw
;
589 struct ioat_ring_ent
*next
;
591 /* copy current descriptors to the new ring, dropping the
592 * removed descriptors
594 for (i
= 0; i
< new_size
; i
++) {
595 u16 curr_idx
= (ioat
->tail
+i
) & (curr_size
-1);
596 u16 new_idx
= (ioat
->tail
+i
) & (new_size
-1);
598 ring
[new_idx
] = ioat
->ring
[curr_idx
];
599 set_desc_id(ring
[new_idx
], new_idx
);
602 /* free deleted descriptors */
603 for (i
= new_size
; i
< curr_size
; i
++) {
604 struct ioat_ring_ent
*ent
;
606 ent
= ioat2_get_ring_ent(ioat
, ioat
->tail
+i
);
607 ioat2_free_ring_ent(ent
, c
);
610 /* fix up hardware ring */
611 hw
= ring
[(ioat
->tail
+new_size
-1) & (new_size
-1)]->hw
;
612 next
= ring
[(ioat
->tail
+new_size
) & (new_size
-1)];
613 hw
->next
= next
->txd
.phys
;
616 dev_dbg(to_dev(chan
), "%s: allocated %d descriptors\n",
621 ioat
->alloc_order
= order
;
627 * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
628 * @idx: gets starting descriptor index on successful allocation
629 * @ioat: ioat2,3 channel (ring) to operate on
630 * @num_descs: allocation length
632 int ioat2_alloc_and_lock(u16
*idx
, struct ioat2_dma_chan
*ioat
, int num_descs
)
634 struct ioat_chan_common
*chan
= &ioat
->base
;
636 spin_lock_bh(&ioat
->ring_lock
);
637 /* never allow the last descriptor to be consumed, we need at
638 * least one free at all times to allow for on-the-fly ring
641 while (unlikely(ioat2_ring_space(ioat
) <= num_descs
)) {
642 if (reshape_ring(ioat
, ioat
->alloc_order
+ 1) &&
643 ioat2_ring_space(ioat
) > num_descs
)
646 if (printk_ratelimit())
647 dev_dbg(to_dev(chan
),
648 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
649 __func__
, num_descs
, ioat
->head
, ioat
->tail
,
651 spin_unlock_bh(&ioat
->ring_lock
);
653 /* progress reclaim in the allocation failure case we
654 * may be called under bh_disabled so we need to trigger
655 * the timer event directly
657 spin_lock_bh(&chan
->cleanup_lock
);
658 if (jiffies
> chan
->timer
.expires
&&
659 timer_pending(&chan
->timer
)) {
660 struct ioatdma_device
*device
= chan
->device
;
662 mod_timer(&chan
->timer
, jiffies
+ COMPLETION_TIMEOUT
);
663 spin_unlock_bh(&chan
->cleanup_lock
);
664 device
->timer_fn((unsigned long) ioat
);
666 spin_unlock_bh(&chan
->cleanup_lock
);
670 dev_dbg(to_dev(chan
), "%s: num_descs: %d (%x:%x:%x)\n",
671 __func__
, num_descs
, ioat
->head
, ioat
->tail
, ioat
->issued
);
673 *idx
= ioat2_desc_alloc(ioat
, num_descs
);
674 return 0; /* with ioat->ring_lock held */
677 struct dma_async_tx_descriptor
*
678 ioat2_dma_prep_memcpy_lock(struct dma_chan
*c
, dma_addr_t dma_dest
,
679 dma_addr_t dma_src
, size_t len
, unsigned long flags
)
681 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
682 struct ioat_dma_descriptor
*hw
;
683 struct ioat_ring_ent
*desc
;
684 dma_addr_t dst
= dma_dest
;
685 dma_addr_t src
= dma_src
;
686 size_t total_len
= len
;
691 num_descs
= ioat2_xferlen_to_descs(ioat
, len
);
692 if (likely(num_descs
) &&
693 ioat2_alloc_and_lock(&idx
, ioat
, num_descs
) == 0)
699 size_t copy
= min_t(size_t, len
, 1 << ioat
->xfercap_log
);
701 desc
= ioat2_get_ring_ent(ioat
, idx
+ i
);
712 dump_desc_dbg(ioat
, desc
);
713 } while (++i
< num_descs
);
715 desc
->txd
.flags
= flags
;
716 desc
->len
= total_len
;
717 hw
->ctl_f
.int_en
= !!(flags
& DMA_PREP_INTERRUPT
);
718 hw
->ctl_f
.fence
= !!(flags
& DMA_PREP_FENCE
);
719 hw
->ctl_f
.compl_write
= 1;
720 dump_desc_dbg(ioat
, desc
);
721 /* we leave the channel locked to ensure in order submission */
727 * ioat2_free_chan_resources - release all the descriptors
728 * @chan: the channel to be cleaned
730 void ioat2_free_chan_resources(struct dma_chan
*c
)
732 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
733 struct ioat_chan_common
*chan
= &ioat
->base
;
734 struct ioatdma_device
*device
= chan
->device
;
735 struct ioat_ring_ent
*desc
;
736 const u16 total_descs
= 1 << ioat
->alloc_order
;
740 /* Before freeing channel resources first check
741 * if they have been previously allocated for this channel.
746 tasklet_disable(&chan
->cleanup_task
);
747 del_timer_sync(&chan
->timer
);
748 device
->cleanup_tasklet((unsigned long) ioat
);
750 /* Delay 100ms after reset to allow internal DMA logic to quiesce
751 * before removing DMA descriptor resources.
753 writeb(IOAT_CHANCMD_RESET
,
754 chan
->reg_base
+ IOAT_CHANCMD_OFFSET(chan
->device
->version
));
757 spin_lock_bh(&ioat
->ring_lock
);
758 descs
= ioat2_ring_space(ioat
);
759 dev_dbg(to_dev(chan
), "freeing %d idle descriptors\n", descs
);
760 for (i
= 0; i
< descs
; i
++) {
761 desc
= ioat2_get_ring_ent(ioat
, ioat
->head
+ i
);
762 ioat2_free_ring_ent(desc
, c
);
765 if (descs
< total_descs
)
766 dev_err(to_dev(chan
), "Freeing %d in use descriptors!\n",
767 total_descs
- descs
);
769 for (i
= 0; i
< total_descs
- descs
; i
++) {
770 desc
= ioat2_get_ring_ent(ioat
, ioat
->tail
+ i
);
771 dump_desc_dbg(ioat
, desc
);
772 ioat2_free_ring_ent(desc
, c
);
777 ioat
->alloc_order
= 0;
778 pci_pool_free(device
->completion_pool
, chan
->completion
,
779 chan
->completion_dma
);
780 spin_unlock_bh(&ioat
->ring_lock
);
782 chan
->last_completion
= 0;
783 chan
->completion_dma
= 0;
789 ioat2_is_complete(struct dma_chan
*c
, dma_cookie_t cookie
,
790 dma_cookie_t
*done
, dma_cookie_t
*used
)
792 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
793 struct ioatdma_device
*device
= ioat
->base
.device
;
795 if (ioat_is_complete(c
, cookie
, done
, used
) == DMA_SUCCESS
)
798 device
->cleanup_tasklet((unsigned long) ioat
);
800 return ioat_is_complete(c
, cookie
, done
, used
);
803 static ssize_t
ring_size_show(struct dma_chan
*c
, char *page
)
805 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
807 return sprintf(page
, "%d\n", (1 << ioat
->alloc_order
) & ~1);
809 static struct ioat_sysfs_entry ring_size_attr
= __ATTR_RO(ring_size
);
811 static ssize_t
ring_active_show(struct dma_chan
*c
, char *page
)
813 struct ioat2_dma_chan
*ioat
= to_ioat2_chan(c
);
815 /* ...taken outside the lock, no need to be precise */
816 return sprintf(page
, "%d\n", ioat2_ring_active(ioat
));
818 static struct ioat_sysfs_entry ring_active_attr
= __ATTR_RO(ring_active
);
820 static struct attribute
*ioat2_attrs
[] = {
821 &ring_size_attr
.attr
,
822 &ring_active_attr
.attr
,
824 &ioat_version_attr
.attr
,
828 struct kobj_type ioat2_ktype
= {
829 .sysfs_ops
= &ioat_sysfs_ops
,
830 .default_attrs
= ioat2_attrs
,
833 int __devinit
ioat2_dma_probe(struct ioatdma_device
*device
, int dca
)
835 struct pci_dev
*pdev
= device
->pdev
;
836 struct dma_device
*dma
;
838 struct ioat_chan_common
*chan
;
841 device
->enumerate_channels
= ioat2_enumerate_channels
;
842 device
->cleanup_tasklet
= ioat2_cleanup_tasklet
;
843 device
->timer_fn
= ioat2_timer_event
;
844 device
->self_test
= ioat_dma_self_test
;
845 dma
= &device
->common
;
846 dma
->device_prep_dma_memcpy
= ioat2_dma_prep_memcpy_lock
;
847 dma
->device_issue_pending
= ioat2_issue_pending
;
848 dma
->device_alloc_chan_resources
= ioat2_alloc_chan_resources
;
849 dma
->device_free_chan_resources
= ioat2_free_chan_resources
;
850 dma
->device_is_tx_complete
= ioat2_is_complete
;
852 err
= ioat_probe(device
);
855 ioat_set_tcp_copy_break(2048);
857 list_for_each_entry(c
, &dma
->channels
, device_node
) {
858 chan
= to_chan_common(c
);
859 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE
| IOAT_DMA_DCA_ANY_CPU
,
860 chan
->reg_base
+ IOAT_DCACTRL_OFFSET
);
863 err
= ioat_register(device
);
867 ioat_kobject_add(device
, &ioat2_ktype
);
870 device
->dca
= ioat2_dca_init(pdev
, device
->reg_base
);