2 * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
5 * Copyright (C) 2007-2008 Atmel Corporation
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 #include <linux/clk.h>
12 #include <linux/delay.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
23 #include "dw_dmac_regs.h"
26 * This supports the Synopsys "DesignWare AHB Central DMA Controller",
27 * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
28 * of which use ARM any more). See the "Databook" from Synopsys for
29 * information beyond what licensees probably provide.
31 * The driver has currently been tested only with the Atmel AT32AP7000,
32 * which does not support descriptor writeback.
35 /* NOTE: DMS+SMS is system-specific. We should get this information
36 * from the platform code somehow.
38 #define DWC_DEFAULT_CTLLO (DWC_CTLL_DST_MSIZE(0) \
39 | DWC_CTLL_SRC_MSIZE(0) \
46 * This is configuration-dependent and usually a funny size like 4095.
47 * Let's round it down to the nearest power of two.
49 * Note that this is a transfer count, i.e. if we transfer 32-bit
50 * words, we can do 8192 bytes per descriptor.
52 * This parameter is also system-specific.
54 #define DWC_MAX_COUNT 2048U
57 * Number of descriptors to allocate for each channel. This should be
58 * made configurable somehow; preferably, the clients (at least the
59 * ones using slave transfers) should be able to give us a hint.
61 #define NR_DESCS_PER_CHANNEL 64
63 /*----------------------------------------------------------------------*/
66 * Because we're not relying on writeback from the controller (it may not
67 * even be configured into the core!) we don't need to use dma_pool. These
68 * descriptors -- and associated data -- are cacheable. We do need to make
69 * sure their dcache entries are written back before handing them off to
70 * the controller, though.
73 static struct device
*chan2dev(struct dma_chan
*chan
)
75 return &chan
->dev
->device
;
77 static struct device
*chan2parent(struct dma_chan
*chan
)
79 return chan
->dev
->device
.parent
;
82 static struct dw_desc
*dwc_first_active(struct dw_dma_chan
*dwc
)
84 return list_entry(dwc
->active_list
.next
, struct dw_desc
, desc_node
);
87 static struct dw_desc
*dwc_first_queued(struct dw_dma_chan
*dwc
)
89 return list_entry(dwc
->queue
.next
, struct dw_desc
, desc_node
);
92 static struct dw_desc
*dwc_desc_get(struct dw_dma_chan
*dwc
)
94 struct dw_desc
*desc
, *_desc
;
95 struct dw_desc
*ret
= NULL
;
98 spin_lock_bh(&dwc
->lock
);
99 list_for_each_entry_safe(desc
, _desc
, &dwc
->free_list
, desc_node
) {
100 if (async_tx_test_ack(&desc
->txd
)) {
101 list_del(&desc
->desc_node
);
105 dev_dbg(chan2dev(&dwc
->chan
), "desc %p not ACKed\n", desc
);
108 spin_unlock_bh(&dwc
->lock
);
110 dev_vdbg(chan2dev(&dwc
->chan
), "scanned %u descriptors on freelist\n", i
);
115 static void dwc_sync_desc_for_cpu(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
117 struct dw_desc
*child
;
119 list_for_each_entry(child
, &desc
->tx_list
, desc_node
)
120 dma_sync_single_for_cpu(chan2parent(&dwc
->chan
),
121 child
->txd
.phys
, sizeof(child
->lli
),
123 dma_sync_single_for_cpu(chan2parent(&dwc
->chan
),
124 desc
->txd
.phys
, sizeof(desc
->lli
),
129 * Move a descriptor, including any children, to the free list.
130 * `desc' must not be on any lists.
132 static void dwc_desc_put(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
135 struct dw_desc
*child
;
137 dwc_sync_desc_for_cpu(dwc
, desc
);
139 spin_lock_bh(&dwc
->lock
);
140 list_for_each_entry(child
, &desc
->tx_list
, desc_node
)
141 dev_vdbg(chan2dev(&dwc
->chan
),
142 "moving child desc %p to freelist\n",
144 list_splice_init(&desc
->tx_list
, &dwc
->free_list
);
145 dev_vdbg(chan2dev(&dwc
->chan
), "moving desc %p to freelist\n", desc
);
146 list_add(&desc
->desc_node
, &dwc
->free_list
);
147 spin_unlock_bh(&dwc
->lock
);
151 /* Called with dwc->lock held and bh disabled */
153 dwc_assign_cookie(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
155 dma_cookie_t cookie
= dwc
->chan
.cookie
;
160 dwc
->chan
.cookie
= cookie
;
161 desc
->txd
.cookie
= cookie
;
166 /*----------------------------------------------------------------------*/
168 /* Called with dwc->lock held and bh disabled */
169 static void dwc_dostart(struct dw_dma_chan
*dwc
, struct dw_desc
*first
)
171 struct dw_dma
*dw
= to_dw_dma(dwc
->chan
.device
);
173 /* ASSERT: channel is idle */
174 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
175 dev_err(chan2dev(&dwc
->chan
),
176 "BUG: Attempted to start non-idle channel\n");
177 dev_err(chan2dev(&dwc
->chan
),
178 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
179 channel_readl(dwc
, SAR
),
180 channel_readl(dwc
, DAR
),
181 channel_readl(dwc
, LLP
),
182 channel_readl(dwc
, CTL_HI
),
183 channel_readl(dwc
, CTL_LO
));
185 /* The tasklet will hopefully advance the queue... */
189 channel_writel(dwc
, LLP
, first
->txd
.phys
);
190 channel_writel(dwc
, CTL_LO
,
191 DWC_CTLL_LLP_D_EN
| DWC_CTLL_LLP_S_EN
);
192 channel_writel(dwc
, CTL_HI
, 0);
193 channel_set_bit(dw
, CH_EN
, dwc
->mask
);
196 /*----------------------------------------------------------------------*/
199 dwc_descriptor_complete(struct dw_dma_chan
*dwc
, struct dw_desc
*desc
)
201 dma_async_tx_callback callback
;
203 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
205 dev_vdbg(chan2dev(&dwc
->chan
), "descriptor %u complete\n", txd
->cookie
);
207 dwc
->completed
= txd
->cookie
;
208 callback
= txd
->callback
;
209 param
= txd
->callback_param
;
211 dwc_sync_desc_for_cpu(dwc
, desc
);
212 list_splice_init(&desc
->tx_list
, &dwc
->free_list
);
213 list_move(&desc
->desc_node
, &dwc
->free_list
);
215 if (!dwc
->chan
.private) {
216 struct device
*parent
= chan2parent(&dwc
->chan
);
217 if (!(txd
->flags
& DMA_COMPL_SKIP_DEST_UNMAP
)) {
218 if (txd
->flags
& DMA_COMPL_DEST_UNMAP_SINGLE
)
219 dma_unmap_single(parent
, desc
->lli
.dar
,
220 desc
->len
, DMA_FROM_DEVICE
);
222 dma_unmap_page(parent
, desc
->lli
.dar
,
223 desc
->len
, DMA_FROM_DEVICE
);
225 if (!(txd
->flags
& DMA_COMPL_SKIP_SRC_UNMAP
)) {
226 if (txd
->flags
& DMA_COMPL_SRC_UNMAP_SINGLE
)
227 dma_unmap_single(parent
, desc
->lli
.sar
,
228 desc
->len
, DMA_TO_DEVICE
);
230 dma_unmap_page(parent
, desc
->lli
.sar
,
231 desc
->len
, DMA_TO_DEVICE
);
236 * The API requires that no submissions are done from a
237 * callback, so we don't need to drop the lock here
243 static void dwc_complete_all(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
)
245 struct dw_desc
*desc
, *_desc
;
248 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
249 dev_err(chan2dev(&dwc
->chan
),
250 "BUG: XFER bit set, but channel not idle!\n");
252 /* Try to continue after resetting the channel... */
253 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
254 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
259 * Submit queued descriptors ASAP, i.e. before we go through
260 * the completed ones.
262 if (!list_empty(&dwc
->queue
))
263 dwc_dostart(dwc
, dwc_first_queued(dwc
));
264 list_splice_init(&dwc
->active_list
, &list
);
265 list_splice_init(&dwc
->queue
, &dwc
->active_list
);
267 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
)
268 dwc_descriptor_complete(dwc
, desc
);
271 static void dwc_scan_descriptors(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
)
274 struct dw_desc
*desc
, *_desc
;
275 struct dw_desc
*child
;
279 * Clear block interrupt flag before scanning so that we don't
280 * miss any, and read LLP before RAW_XFER to ensure it is
281 * valid if we decide to scan the list.
283 dma_writel(dw
, CLEAR
.BLOCK
, dwc
->mask
);
284 llp
= channel_readl(dwc
, LLP
);
285 status_xfer
= dma_readl(dw
, RAW
.XFER
);
287 if (status_xfer
& dwc
->mask
) {
288 /* Everything we've submitted is done */
289 dma_writel(dw
, CLEAR
.XFER
, dwc
->mask
);
290 dwc_complete_all(dw
, dwc
);
294 dev_vdbg(chan2dev(&dwc
->chan
), "scan_descriptors: llp=0x%x\n", llp
);
296 list_for_each_entry_safe(desc
, _desc
, &dwc
->active_list
, desc_node
) {
297 if (desc
->lli
.llp
== llp
)
298 /* This one is currently in progress */
301 list_for_each_entry(child
, &desc
->tx_list
, desc_node
)
302 if (child
->lli
.llp
== llp
)
303 /* Currently in progress */
307 * No descriptors so far seem to be in progress, i.e.
308 * this one must be done.
310 dwc_descriptor_complete(dwc
, desc
);
313 dev_err(chan2dev(&dwc
->chan
),
314 "BUG: All descriptors done, but channel not idle!\n");
316 /* Try to continue after resetting the channel... */
317 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
318 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
321 if (!list_empty(&dwc
->queue
)) {
322 dwc_dostart(dwc
, dwc_first_queued(dwc
));
323 list_splice_init(&dwc
->queue
, &dwc
->active_list
);
327 static void dwc_dump_lli(struct dw_dma_chan
*dwc
, struct dw_lli
*lli
)
329 dev_printk(KERN_CRIT
, chan2dev(&dwc
->chan
),
330 " desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
331 lli
->sar
, lli
->dar
, lli
->llp
,
332 lli
->ctlhi
, lli
->ctllo
);
335 static void dwc_handle_error(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
)
337 struct dw_desc
*bad_desc
;
338 struct dw_desc
*child
;
340 dwc_scan_descriptors(dw
, dwc
);
343 * The descriptor currently at the head of the active list is
344 * borked. Since we don't have any way to report errors, we'll
345 * just have to scream loudly and try to carry on.
347 bad_desc
= dwc_first_active(dwc
);
348 list_del_init(&bad_desc
->desc_node
);
349 list_splice_init(&dwc
->queue
, dwc
->active_list
.prev
);
351 /* Clear the error flag and try to restart the controller */
352 dma_writel(dw
, CLEAR
.ERROR
, dwc
->mask
);
353 if (!list_empty(&dwc
->active_list
))
354 dwc_dostart(dwc
, dwc_first_active(dwc
));
357 * KERN_CRITICAL may seem harsh, but since this only happens
358 * when someone submits a bad physical address in a
359 * descriptor, we should consider ourselves lucky that the
360 * controller flagged an error instead of scribbling over
361 * random memory locations.
363 dev_printk(KERN_CRIT
, chan2dev(&dwc
->chan
),
364 "Bad descriptor submitted for DMA!\n");
365 dev_printk(KERN_CRIT
, chan2dev(&dwc
->chan
),
366 " cookie: %d\n", bad_desc
->txd
.cookie
);
367 dwc_dump_lli(dwc
, &bad_desc
->lli
);
368 list_for_each_entry(child
, &bad_desc
->tx_list
, desc_node
)
369 dwc_dump_lli(dwc
, &child
->lli
);
371 /* Pretend the descriptor completed successfully */
372 dwc_descriptor_complete(dwc
, bad_desc
);
375 /* --------------------- Cyclic DMA API extensions -------------------- */
377 inline dma_addr_t
dw_dma_get_src_addr(struct dma_chan
*chan
)
379 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
380 return channel_readl(dwc
, SAR
);
382 EXPORT_SYMBOL(dw_dma_get_src_addr
);
384 inline dma_addr_t
dw_dma_get_dst_addr(struct dma_chan
*chan
)
386 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
387 return channel_readl(dwc
, DAR
);
389 EXPORT_SYMBOL(dw_dma_get_dst_addr
);
391 /* called with dwc->lock held and all DMAC interrupts disabled */
392 static void dwc_handle_cyclic(struct dw_dma
*dw
, struct dw_dma_chan
*dwc
,
393 u32 status_block
, u32 status_err
, u32 status_xfer
)
395 if (status_block
& dwc
->mask
) {
396 void (*callback
)(void *param
);
397 void *callback_param
;
399 dev_vdbg(chan2dev(&dwc
->chan
), "new cyclic period llp 0x%08x\n",
400 channel_readl(dwc
, LLP
));
401 dma_writel(dw
, CLEAR
.BLOCK
, dwc
->mask
);
403 callback
= dwc
->cdesc
->period_callback
;
404 callback_param
= dwc
->cdesc
->period_callback_param
;
406 spin_unlock(&dwc
->lock
);
407 callback(callback_param
);
408 spin_lock(&dwc
->lock
);
413 * Error and transfer complete are highly unlikely, and will most
414 * likely be due to a configuration error by the user.
416 if (unlikely(status_err
& dwc
->mask
) ||
417 unlikely(status_xfer
& dwc
->mask
)) {
420 dev_err(chan2dev(&dwc
->chan
), "cyclic DMA unexpected %s "
421 "interrupt, stopping DMA transfer\n",
422 status_xfer
? "xfer" : "error");
423 dev_err(chan2dev(&dwc
->chan
),
424 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
425 channel_readl(dwc
, SAR
),
426 channel_readl(dwc
, DAR
),
427 channel_readl(dwc
, LLP
),
428 channel_readl(dwc
, CTL_HI
),
429 channel_readl(dwc
, CTL_LO
));
431 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
432 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
435 /* make sure DMA does not restart by loading a new list */
436 channel_writel(dwc
, LLP
, 0);
437 channel_writel(dwc
, CTL_LO
, 0);
438 channel_writel(dwc
, CTL_HI
, 0);
440 dma_writel(dw
, CLEAR
.BLOCK
, dwc
->mask
);
441 dma_writel(dw
, CLEAR
.ERROR
, dwc
->mask
);
442 dma_writel(dw
, CLEAR
.XFER
, dwc
->mask
);
444 for (i
= 0; i
< dwc
->cdesc
->periods
; i
++)
445 dwc_dump_lli(dwc
, &dwc
->cdesc
->desc
[i
]->lli
);
449 /* ------------------------------------------------------------------------- */
451 static void dw_dma_tasklet(unsigned long data
)
453 struct dw_dma
*dw
= (struct dw_dma
*)data
;
454 struct dw_dma_chan
*dwc
;
460 status_block
= dma_readl(dw
, RAW
.BLOCK
);
461 status_xfer
= dma_readl(dw
, RAW
.XFER
);
462 status_err
= dma_readl(dw
, RAW
.ERROR
);
464 dev_vdbg(dw
->dma
.dev
, "tasklet: status_block=%x status_err=%x\n",
465 status_block
, status_err
);
467 for (i
= 0; i
< dw
->dma
.chancnt
; i
++) {
469 spin_lock(&dwc
->lock
);
470 if (test_bit(DW_DMA_IS_CYCLIC
, &dwc
->flags
))
471 dwc_handle_cyclic(dw
, dwc
, status_block
, status_err
,
473 else if (status_err
& (1 << i
))
474 dwc_handle_error(dw
, dwc
);
475 else if ((status_block
| status_xfer
) & (1 << i
))
476 dwc_scan_descriptors(dw
, dwc
);
477 spin_unlock(&dwc
->lock
);
481 * Re-enable interrupts. Block Complete interrupts are only
482 * enabled if the INT_EN bit in the descriptor is set. This
483 * will trigger a scan before the whole list is done.
485 channel_set_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
486 channel_set_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
487 channel_set_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
490 static irqreturn_t
dw_dma_interrupt(int irq
, void *dev_id
)
492 struct dw_dma
*dw
= dev_id
;
495 dev_vdbg(dw
->dma
.dev
, "interrupt: status=0x%x\n",
496 dma_readl(dw
, STATUS_INT
));
499 * Just disable the interrupts. We'll turn them back on in the
502 channel_clear_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
503 channel_clear_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
504 channel_clear_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
506 status
= dma_readl(dw
, STATUS_INT
);
509 "BUG: Unexpected interrupts pending: 0x%x\n",
513 channel_clear_bit(dw
, MASK
.XFER
, (1 << 8) - 1);
514 channel_clear_bit(dw
, MASK
.BLOCK
, (1 << 8) - 1);
515 channel_clear_bit(dw
, MASK
.SRC_TRAN
, (1 << 8) - 1);
516 channel_clear_bit(dw
, MASK
.DST_TRAN
, (1 << 8) - 1);
517 channel_clear_bit(dw
, MASK
.ERROR
, (1 << 8) - 1);
520 tasklet_schedule(&dw
->tasklet
);
525 /*----------------------------------------------------------------------*/
527 static dma_cookie_t
dwc_tx_submit(struct dma_async_tx_descriptor
*tx
)
529 struct dw_desc
*desc
= txd_to_dw_desc(tx
);
530 struct dw_dma_chan
*dwc
= to_dw_dma_chan(tx
->chan
);
533 spin_lock_bh(&dwc
->lock
);
534 cookie
= dwc_assign_cookie(dwc
, desc
);
537 * REVISIT: We should attempt to chain as many descriptors as
538 * possible, perhaps even appending to those already submitted
539 * for DMA. But this is hard to do in a race-free manner.
541 if (list_empty(&dwc
->active_list
)) {
542 dev_vdbg(chan2dev(tx
->chan
), "tx_submit: started %u\n",
544 dwc_dostart(dwc
, desc
);
545 list_add_tail(&desc
->desc_node
, &dwc
->active_list
);
547 dev_vdbg(chan2dev(tx
->chan
), "tx_submit: queued %u\n",
550 list_add_tail(&desc
->desc_node
, &dwc
->queue
);
553 spin_unlock_bh(&dwc
->lock
);
558 static struct dma_async_tx_descriptor
*
559 dwc_prep_dma_memcpy(struct dma_chan
*chan
, dma_addr_t dest
, dma_addr_t src
,
560 size_t len
, unsigned long flags
)
562 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
563 struct dw_desc
*desc
;
564 struct dw_desc
*first
;
565 struct dw_desc
*prev
;
568 unsigned int src_width
;
569 unsigned int dst_width
;
572 dev_vdbg(chan2dev(chan
), "prep_dma_memcpy d0x%x s0x%x l0x%zx f0x%lx\n",
573 dest
, src
, len
, flags
);
575 if (unlikely(!len
)) {
576 dev_dbg(chan2dev(chan
), "prep_dma_memcpy: length is zero!\n");
581 * We can be a lot more clever here, but this should take care
582 * of the most common optimization.
584 if (!((src
| dest
| len
) & 3))
585 src_width
= dst_width
= 2;
586 else if (!((src
| dest
| len
) & 1))
587 src_width
= dst_width
= 1;
589 src_width
= dst_width
= 0;
591 ctllo
= DWC_DEFAULT_CTLLO
592 | DWC_CTLL_DST_WIDTH(dst_width
)
593 | DWC_CTLL_SRC_WIDTH(src_width
)
599 for (offset
= 0; offset
< len
; offset
+= xfer_count
<< src_width
) {
600 xfer_count
= min_t(size_t, (len
- offset
) >> src_width
,
603 desc
= dwc_desc_get(dwc
);
607 desc
->lli
.sar
= src
+ offset
;
608 desc
->lli
.dar
= dest
+ offset
;
609 desc
->lli
.ctllo
= ctllo
;
610 desc
->lli
.ctlhi
= xfer_count
;
615 prev
->lli
.llp
= desc
->txd
.phys
;
616 dma_sync_single_for_device(chan2parent(chan
),
617 prev
->txd
.phys
, sizeof(prev
->lli
),
619 list_add_tail(&desc
->desc_node
,
626 if (flags
& DMA_PREP_INTERRUPT
)
627 /* Trigger interrupt after last block */
628 prev
->lli
.ctllo
|= DWC_CTLL_INT_EN
;
631 dma_sync_single_for_device(chan2parent(chan
),
632 prev
->txd
.phys
, sizeof(prev
->lli
),
635 first
->txd
.flags
= flags
;
641 dwc_desc_put(dwc
, first
);
645 static struct dma_async_tx_descriptor
*
646 dwc_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
647 unsigned int sg_len
, enum dma_data_direction direction
,
650 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
651 struct dw_dma_slave
*dws
= chan
->private;
652 struct dw_desc
*prev
;
653 struct dw_desc
*first
;
656 unsigned int reg_width
;
657 unsigned int mem_width
;
659 struct scatterlist
*sg
;
660 size_t total_len
= 0;
662 dev_vdbg(chan2dev(chan
), "prep_dma_slave\n");
664 if (unlikely(!dws
|| !sg_len
))
667 reg_width
= dws
->reg_width
;
672 ctllo
= (DWC_DEFAULT_CTLLO
673 | DWC_CTLL_DST_WIDTH(reg_width
)
678 for_each_sg(sgl
, sg
, sg_len
, i
) {
679 struct dw_desc
*desc
;
683 desc
= dwc_desc_get(dwc
);
685 dev_err(chan2dev(chan
),
686 "not enough descriptors available\n");
691 len
= sg_dma_len(sg
);
693 if (unlikely(mem
& 3 || len
& 3))
698 desc
->lli
.ctllo
= ctllo
| DWC_CTLL_SRC_WIDTH(mem_width
);
699 desc
->lli
.ctlhi
= len
>> mem_width
;
704 prev
->lli
.llp
= desc
->txd
.phys
;
705 dma_sync_single_for_device(chan2parent(chan
),
709 list_add_tail(&desc
->desc_node
,
716 case DMA_FROM_DEVICE
:
717 ctllo
= (DWC_DEFAULT_CTLLO
718 | DWC_CTLL_SRC_WIDTH(reg_width
)
724 for_each_sg(sgl
, sg
, sg_len
, i
) {
725 struct dw_desc
*desc
;
729 desc
= dwc_desc_get(dwc
);
731 dev_err(chan2dev(chan
),
732 "not enough descriptors available\n");
737 len
= sg_dma_len(sg
);
739 if (unlikely(mem
& 3 || len
& 3))
744 desc
->lli
.ctllo
= ctllo
| DWC_CTLL_DST_WIDTH(mem_width
);
745 desc
->lli
.ctlhi
= len
>> reg_width
;
750 prev
->lli
.llp
= desc
->txd
.phys
;
751 dma_sync_single_for_device(chan2parent(chan
),
755 list_add_tail(&desc
->desc_node
,
766 if (flags
& DMA_PREP_INTERRUPT
)
767 /* Trigger interrupt after last block */
768 prev
->lli
.ctllo
|= DWC_CTLL_INT_EN
;
771 dma_sync_single_for_device(chan2parent(chan
),
772 prev
->txd
.phys
, sizeof(prev
->lli
),
775 first
->len
= total_len
;
780 dwc_desc_put(dwc
, first
);
784 static int dwc_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
)
786 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
787 struct dw_dma
*dw
= to_dw_dma(chan
->device
);
788 struct dw_desc
*desc
, *_desc
;
791 /* Only supports DMA_TERMINATE_ALL */
792 if (cmd
!= DMA_TERMINATE_ALL
)
796 * This is only called when something went wrong elsewhere, so
797 * we don't really care about the data. Just disable the
798 * channel. We still have to poll the channel enable bit due
799 * to AHB/HSB limitations.
801 spin_lock_bh(&dwc
->lock
);
803 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
805 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
808 /* active_list entries will end up before queued entries */
809 list_splice_init(&dwc
->queue
, &list
);
810 list_splice_init(&dwc
->active_list
, &list
);
812 spin_unlock_bh(&dwc
->lock
);
814 /* Flush all pending and queued descriptors */
815 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
)
816 dwc_descriptor_complete(dwc
, desc
);
821 static enum dma_status
822 dwc_is_tx_complete(struct dma_chan
*chan
,
824 dma_cookie_t
*done
, dma_cookie_t
*used
)
826 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
827 dma_cookie_t last_used
;
828 dma_cookie_t last_complete
;
831 last_complete
= dwc
->completed
;
832 last_used
= chan
->cookie
;
834 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
835 if (ret
!= DMA_SUCCESS
) {
836 dwc_scan_descriptors(to_dw_dma(chan
->device
), dwc
);
838 last_complete
= dwc
->completed
;
839 last_used
= chan
->cookie
;
841 ret
= dma_async_is_complete(cookie
, last_complete
, last_used
);
845 *done
= last_complete
;
852 static void dwc_issue_pending(struct dma_chan
*chan
)
854 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
856 spin_lock_bh(&dwc
->lock
);
857 if (!list_empty(&dwc
->queue
))
858 dwc_scan_descriptors(to_dw_dma(chan
->device
), dwc
);
859 spin_unlock_bh(&dwc
->lock
);
862 static int dwc_alloc_chan_resources(struct dma_chan
*chan
)
864 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
865 struct dw_dma
*dw
= to_dw_dma(chan
->device
);
866 struct dw_desc
*desc
;
867 struct dw_dma_slave
*dws
;
872 dev_vdbg(chan2dev(chan
), "alloc_chan_resources\n");
874 /* ASSERT: channel is idle */
875 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
876 dev_dbg(chan2dev(chan
), "DMA channel not idle?\n");
880 dwc
->completed
= chan
->cookie
= 1;
882 cfghi
= DWC_CFGH_FIFO_MODE
;
888 * We need controller-specific data to set up slave
891 BUG_ON(!dws
->dma_dev
|| dws
->dma_dev
!= dw
->dma
.dev
);
896 channel_writel(dwc
, CFG_LO
, cfglo
);
897 channel_writel(dwc
, CFG_HI
, cfghi
);
900 * NOTE: some controllers may have additional features that we
901 * need to initialize here, like "scatter-gather" (which
902 * doesn't mean what you think it means), and status writeback.
905 spin_lock_bh(&dwc
->lock
);
906 i
= dwc
->descs_allocated
;
907 while (dwc
->descs_allocated
< NR_DESCS_PER_CHANNEL
) {
908 spin_unlock_bh(&dwc
->lock
);
910 desc
= kzalloc(sizeof(struct dw_desc
), GFP_KERNEL
);
912 dev_info(chan2dev(chan
),
913 "only allocated %d descriptors\n", i
);
914 spin_lock_bh(&dwc
->lock
);
918 INIT_LIST_HEAD(&desc
->tx_list
);
919 dma_async_tx_descriptor_init(&desc
->txd
, chan
);
920 desc
->txd
.tx_submit
= dwc_tx_submit
;
921 desc
->txd
.flags
= DMA_CTRL_ACK
;
922 desc
->txd
.phys
= dma_map_single(chan2parent(chan
), &desc
->lli
,
923 sizeof(desc
->lli
), DMA_TO_DEVICE
);
924 dwc_desc_put(dwc
, desc
);
926 spin_lock_bh(&dwc
->lock
);
927 i
= ++dwc
->descs_allocated
;
930 /* Enable interrupts */
931 channel_set_bit(dw
, MASK
.XFER
, dwc
->mask
);
932 channel_set_bit(dw
, MASK
.BLOCK
, dwc
->mask
);
933 channel_set_bit(dw
, MASK
.ERROR
, dwc
->mask
);
935 spin_unlock_bh(&dwc
->lock
);
937 dev_dbg(chan2dev(chan
),
938 "alloc_chan_resources allocated %d descriptors\n", i
);
943 static void dwc_free_chan_resources(struct dma_chan
*chan
)
945 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
946 struct dw_dma
*dw
= to_dw_dma(chan
->device
);
947 struct dw_desc
*desc
, *_desc
;
950 dev_dbg(chan2dev(chan
), "free_chan_resources (descs allocated=%u)\n",
951 dwc
->descs_allocated
);
953 /* ASSERT: channel is idle */
954 BUG_ON(!list_empty(&dwc
->active_list
));
955 BUG_ON(!list_empty(&dwc
->queue
));
956 BUG_ON(dma_readl(to_dw_dma(chan
->device
), CH_EN
) & dwc
->mask
);
958 spin_lock_bh(&dwc
->lock
);
959 list_splice_init(&dwc
->free_list
, &list
);
960 dwc
->descs_allocated
= 0;
962 /* Disable interrupts */
963 channel_clear_bit(dw
, MASK
.XFER
, dwc
->mask
);
964 channel_clear_bit(dw
, MASK
.BLOCK
, dwc
->mask
);
965 channel_clear_bit(dw
, MASK
.ERROR
, dwc
->mask
);
967 spin_unlock_bh(&dwc
->lock
);
969 list_for_each_entry_safe(desc
, _desc
, &list
, desc_node
) {
970 dev_vdbg(chan2dev(chan
), " freeing descriptor %p\n", desc
);
971 dma_unmap_single(chan2parent(chan
), desc
->txd
.phys
,
972 sizeof(desc
->lli
), DMA_TO_DEVICE
);
976 dev_vdbg(chan2dev(chan
), "free_chan_resources done\n");
979 /* --------------------- Cyclic DMA API extensions -------------------- */
982 * dw_dma_cyclic_start - start the cyclic DMA transfer
983 * @chan: the DMA channel to start
985 * Must be called with soft interrupts disabled. Returns zero on success or
988 int dw_dma_cyclic_start(struct dma_chan
*chan
)
990 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
991 struct dw_dma
*dw
= to_dw_dma(dwc
->chan
.device
);
993 if (!test_bit(DW_DMA_IS_CYCLIC
, &dwc
->flags
)) {
994 dev_err(chan2dev(&dwc
->chan
), "missing prep for cyclic DMA\n");
998 spin_lock(&dwc
->lock
);
1000 /* assert channel is idle */
1001 if (dma_readl(dw
, CH_EN
) & dwc
->mask
) {
1002 dev_err(chan2dev(&dwc
->chan
),
1003 "BUG: Attempted to start non-idle channel\n");
1004 dev_err(chan2dev(&dwc
->chan
),
1005 " SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
1006 channel_readl(dwc
, SAR
),
1007 channel_readl(dwc
, DAR
),
1008 channel_readl(dwc
, LLP
),
1009 channel_readl(dwc
, CTL_HI
),
1010 channel_readl(dwc
, CTL_LO
));
1011 spin_unlock(&dwc
->lock
);
1015 dma_writel(dw
, CLEAR
.BLOCK
, dwc
->mask
);
1016 dma_writel(dw
, CLEAR
.ERROR
, dwc
->mask
);
1017 dma_writel(dw
, CLEAR
.XFER
, dwc
->mask
);
1019 /* setup DMAC channel registers */
1020 channel_writel(dwc
, LLP
, dwc
->cdesc
->desc
[0]->txd
.phys
);
1021 channel_writel(dwc
, CTL_LO
, DWC_CTLL_LLP_D_EN
| DWC_CTLL_LLP_S_EN
);
1022 channel_writel(dwc
, CTL_HI
, 0);
1024 channel_set_bit(dw
, CH_EN
, dwc
->mask
);
1026 spin_unlock(&dwc
->lock
);
1030 EXPORT_SYMBOL(dw_dma_cyclic_start
);
1033 * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1034 * @chan: the DMA channel to stop
1036 * Must be called with soft interrupts disabled.
1038 void dw_dma_cyclic_stop(struct dma_chan
*chan
)
1040 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
1041 struct dw_dma
*dw
= to_dw_dma(dwc
->chan
.device
);
1043 spin_lock(&dwc
->lock
);
1045 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
1046 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
1049 spin_unlock(&dwc
->lock
);
1051 EXPORT_SYMBOL(dw_dma_cyclic_stop
);
1054 * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1055 * @chan: the DMA channel to prepare
1056 * @buf_addr: physical DMA address where the buffer starts
1057 * @buf_len: total number of bytes for the entire buffer
1058 * @period_len: number of bytes for each period
1059 * @direction: transfer direction, to or from device
1061 * Must be called before trying to start the transfer. Returns a valid struct
1062 * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1064 struct dw_cyclic_desc
*dw_dma_cyclic_prep(struct dma_chan
*chan
,
1065 dma_addr_t buf_addr
, size_t buf_len
, size_t period_len
,
1066 enum dma_data_direction direction
)
1068 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
1069 struct dw_cyclic_desc
*cdesc
;
1070 struct dw_cyclic_desc
*retval
= NULL
;
1071 struct dw_desc
*desc
;
1072 struct dw_desc
*last
= NULL
;
1073 struct dw_dma_slave
*dws
= chan
->private;
1074 unsigned long was_cyclic
;
1075 unsigned int reg_width
;
1076 unsigned int periods
;
1079 spin_lock_bh(&dwc
->lock
);
1080 if (!list_empty(&dwc
->queue
) || !list_empty(&dwc
->active_list
)) {
1081 spin_unlock_bh(&dwc
->lock
);
1082 dev_dbg(chan2dev(&dwc
->chan
),
1083 "queue and/or active list are not empty\n");
1084 return ERR_PTR(-EBUSY
);
1087 was_cyclic
= test_and_set_bit(DW_DMA_IS_CYCLIC
, &dwc
->flags
);
1088 spin_unlock_bh(&dwc
->lock
);
1090 dev_dbg(chan2dev(&dwc
->chan
),
1091 "channel already prepared for cyclic DMA\n");
1092 return ERR_PTR(-EBUSY
);
1095 retval
= ERR_PTR(-EINVAL
);
1096 reg_width
= dws
->reg_width
;
1097 periods
= buf_len
/ period_len
;
1099 /* Check for too big/unaligned periods and unaligned DMA buffer. */
1100 if (period_len
> (DWC_MAX_COUNT
<< reg_width
))
1102 if (unlikely(period_len
& ((1 << reg_width
) - 1)))
1104 if (unlikely(buf_addr
& ((1 << reg_width
) - 1)))
1106 if (unlikely(!(direction
& (DMA_TO_DEVICE
| DMA_FROM_DEVICE
))))
1109 retval
= ERR_PTR(-ENOMEM
);
1111 if (periods
> NR_DESCS_PER_CHANNEL
)
1114 cdesc
= kzalloc(sizeof(struct dw_cyclic_desc
), GFP_KERNEL
);
1118 cdesc
->desc
= kzalloc(sizeof(struct dw_desc
*) * periods
, GFP_KERNEL
);
1122 for (i
= 0; i
< periods
; i
++) {
1123 desc
= dwc_desc_get(dwc
);
1125 goto out_err_desc_get
;
1127 switch (direction
) {
1129 desc
->lli
.dar
= dws
->tx_reg
;
1130 desc
->lli
.sar
= buf_addr
+ (period_len
* i
);
1131 desc
->lli
.ctllo
= (DWC_DEFAULT_CTLLO
1132 | DWC_CTLL_DST_WIDTH(reg_width
)
1133 | DWC_CTLL_SRC_WIDTH(reg_width
)
1139 case DMA_FROM_DEVICE
:
1140 desc
->lli
.dar
= buf_addr
+ (period_len
* i
);
1141 desc
->lli
.sar
= dws
->rx_reg
;
1142 desc
->lli
.ctllo
= (DWC_DEFAULT_CTLLO
1143 | DWC_CTLL_SRC_WIDTH(reg_width
)
1144 | DWC_CTLL_DST_WIDTH(reg_width
)
1154 desc
->lli
.ctlhi
= (period_len
>> reg_width
);
1155 cdesc
->desc
[i
] = desc
;
1158 last
->lli
.llp
= desc
->txd
.phys
;
1159 dma_sync_single_for_device(chan2parent(chan
),
1160 last
->txd
.phys
, sizeof(last
->lli
),
1167 /* lets make a cyclic list */
1168 last
->lli
.llp
= cdesc
->desc
[0]->txd
.phys
;
1169 dma_sync_single_for_device(chan2parent(chan
), last
->txd
.phys
,
1170 sizeof(last
->lli
), DMA_TO_DEVICE
);
1172 dev_dbg(chan2dev(&dwc
->chan
), "cyclic prepared buf 0x%08x len %zu "
1173 "period %zu periods %d\n", buf_addr
, buf_len
,
1174 period_len
, periods
);
1176 cdesc
->periods
= periods
;
1183 dwc_desc_put(dwc
, cdesc
->desc
[i
]);
1187 clear_bit(DW_DMA_IS_CYCLIC
, &dwc
->flags
);
1188 return (struct dw_cyclic_desc
*)retval
;
1190 EXPORT_SYMBOL(dw_dma_cyclic_prep
);
1193 * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1194 * @chan: the DMA channel to free
1196 void dw_dma_cyclic_free(struct dma_chan
*chan
)
1198 struct dw_dma_chan
*dwc
= to_dw_dma_chan(chan
);
1199 struct dw_dma
*dw
= to_dw_dma(dwc
->chan
.device
);
1200 struct dw_cyclic_desc
*cdesc
= dwc
->cdesc
;
1203 dev_dbg(chan2dev(&dwc
->chan
), "cyclic free\n");
1208 spin_lock_bh(&dwc
->lock
);
1210 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
1211 while (dma_readl(dw
, CH_EN
) & dwc
->mask
)
1214 dma_writel(dw
, CLEAR
.BLOCK
, dwc
->mask
);
1215 dma_writel(dw
, CLEAR
.ERROR
, dwc
->mask
);
1216 dma_writel(dw
, CLEAR
.XFER
, dwc
->mask
);
1218 spin_unlock_bh(&dwc
->lock
);
1220 for (i
= 0; i
< cdesc
->periods
; i
++)
1221 dwc_desc_put(dwc
, cdesc
->desc
[i
]);
1226 clear_bit(DW_DMA_IS_CYCLIC
, &dwc
->flags
);
1228 EXPORT_SYMBOL(dw_dma_cyclic_free
);
1230 /*----------------------------------------------------------------------*/
1232 static void dw_dma_off(struct dw_dma
*dw
)
1234 dma_writel(dw
, CFG
, 0);
1236 channel_clear_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
1237 channel_clear_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
1238 channel_clear_bit(dw
, MASK
.SRC_TRAN
, dw
->all_chan_mask
);
1239 channel_clear_bit(dw
, MASK
.DST_TRAN
, dw
->all_chan_mask
);
1240 channel_clear_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
1242 while (dma_readl(dw
, CFG
) & DW_CFG_DMA_EN
)
1246 static int __init
dw_probe(struct platform_device
*pdev
)
1248 struct dw_dma_platform_data
*pdata
;
1249 struct resource
*io
;
1256 pdata
= pdev
->dev
.platform_data
;
1257 if (!pdata
|| pdata
->nr_channels
> DW_DMA_MAX_NR_CHANNELS
)
1260 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1264 irq
= platform_get_irq(pdev
, 0);
1268 size
= sizeof(struct dw_dma
);
1269 size
+= pdata
->nr_channels
* sizeof(struct dw_dma_chan
);
1270 dw
= kzalloc(size
, GFP_KERNEL
);
1274 if (!request_mem_region(io
->start
, DW_REGLEN
, pdev
->dev
.driver
->name
)) {
1279 dw
->regs
= ioremap(io
->start
, DW_REGLEN
);
1285 dw
->clk
= clk_get(&pdev
->dev
, "hclk");
1286 if (IS_ERR(dw
->clk
)) {
1287 err
= PTR_ERR(dw
->clk
);
1290 clk_enable(dw
->clk
);
1292 /* force dma off, just in case */
1295 err
= request_irq(irq
, dw_dma_interrupt
, 0, "dw_dmac", dw
);
1299 platform_set_drvdata(pdev
, dw
);
1301 tasklet_init(&dw
->tasklet
, dw_dma_tasklet
, (unsigned long)dw
);
1303 dw
->all_chan_mask
= (1 << pdata
->nr_channels
) - 1;
1305 INIT_LIST_HEAD(&dw
->dma
.channels
);
1306 for (i
= 0; i
< pdata
->nr_channels
; i
++, dw
->dma
.chancnt
++) {
1307 struct dw_dma_chan
*dwc
= &dw
->chan
[i
];
1309 dwc
->chan
.device
= &dw
->dma
;
1310 dwc
->chan
.cookie
= dwc
->completed
= 1;
1311 dwc
->chan
.chan_id
= i
;
1312 list_add_tail(&dwc
->chan
.device_node
, &dw
->dma
.channels
);
1314 dwc
->ch_regs
= &__dw_regs(dw
)->CHAN
[i
];
1315 spin_lock_init(&dwc
->lock
);
1318 INIT_LIST_HEAD(&dwc
->active_list
);
1319 INIT_LIST_HEAD(&dwc
->queue
);
1320 INIT_LIST_HEAD(&dwc
->free_list
);
1322 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
1325 /* Clear/disable all interrupts on all channels. */
1326 dma_writel(dw
, CLEAR
.XFER
, dw
->all_chan_mask
);
1327 dma_writel(dw
, CLEAR
.BLOCK
, dw
->all_chan_mask
);
1328 dma_writel(dw
, CLEAR
.SRC_TRAN
, dw
->all_chan_mask
);
1329 dma_writel(dw
, CLEAR
.DST_TRAN
, dw
->all_chan_mask
);
1330 dma_writel(dw
, CLEAR
.ERROR
, dw
->all_chan_mask
);
1332 channel_clear_bit(dw
, MASK
.XFER
, dw
->all_chan_mask
);
1333 channel_clear_bit(dw
, MASK
.BLOCK
, dw
->all_chan_mask
);
1334 channel_clear_bit(dw
, MASK
.SRC_TRAN
, dw
->all_chan_mask
);
1335 channel_clear_bit(dw
, MASK
.DST_TRAN
, dw
->all_chan_mask
);
1336 channel_clear_bit(dw
, MASK
.ERROR
, dw
->all_chan_mask
);
1338 dma_cap_set(DMA_MEMCPY
, dw
->dma
.cap_mask
);
1339 dma_cap_set(DMA_SLAVE
, dw
->dma
.cap_mask
);
1340 dw
->dma
.dev
= &pdev
->dev
;
1341 dw
->dma
.device_alloc_chan_resources
= dwc_alloc_chan_resources
;
1342 dw
->dma
.device_free_chan_resources
= dwc_free_chan_resources
;
1344 dw
->dma
.device_prep_dma_memcpy
= dwc_prep_dma_memcpy
;
1346 dw
->dma
.device_prep_slave_sg
= dwc_prep_slave_sg
;
1347 dw
->dma
.device_control
= dwc_control
;
1349 dw
->dma
.device_is_tx_complete
= dwc_is_tx_complete
;
1350 dw
->dma
.device_issue_pending
= dwc_issue_pending
;
1352 dma_writel(dw
, CFG
, DW_CFG_DMA_EN
);
1354 printk(KERN_INFO
"%s: DesignWare DMA Controller, %d channels\n",
1355 dev_name(&pdev
->dev
), dw
->dma
.chancnt
);
1357 dma_async_device_register(&dw
->dma
);
1362 clk_disable(dw
->clk
);
1368 release_resource(io
);
1374 static int __exit
dw_remove(struct platform_device
*pdev
)
1376 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1377 struct dw_dma_chan
*dwc
, *_dwc
;
1378 struct resource
*io
;
1381 dma_async_device_unregister(&dw
->dma
);
1383 free_irq(platform_get_irq(pdev
, 0), dw
);
1384 tasklet_kill(&dw
->tasklet
);
1386 list_for_each_entry_safe(dwc
, _dwc
, &dw
->dma
.channels
,
1388 list_del(&dwc
->chan
.device_node
);
1389 channel_clear_bit(dw
, CH_EN
, dwc
->mask
);
1392 clk_disable(dw
->clk
);
1398 io
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1399 release_mem_region(io
->start
, DW_REGLEN
);
1406 static void dw_shutdown(struct platform_device
*pdev
)
1408 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1410 dw_dma_off(platform_get_drvdata(pdev
));
1411 clk_disable(dw
->clk
);
1414 static int dw_suspend_noirq(struct device
*dev
)
1416 struct platform_device
*pdev
= to_platform_device(dev
);
1417 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1419 dw_dma_off(platform_get_drvdata(pdev
));
1420 clk_disable(dw
->clk
);
1424 static int dw_resume_noirq(struct device
*dev
)
1426 struct platform_device
*pdev
= to_platform_device(dev
);
1427 struct dw_dma
*dw
= platform_get_drvdata(pdev
);
1429 clk_enable(dw
->clk
);
1430 dma_writel(dw
, CFG
, DW_CFG_DMA_EN
);
1434 static const struct dev_pm_ops dw_dev_pm_ops
= {
1435 .suspend_noirq
= dw_suspend_noirq
,
1436 .resume_noirq
= dw_resume_noirq
,
1439 static struct platform_driver dw_driver
= {
1440 .remove
= __exit_p(dw_remove
),
1441 .shutdown
= dw_shutdown
,
1444 .pm
= &dw_dev_pm_ops
,
1448 static int __init
dw_init(void)
1450 return platform_driver_probe(&dw_driver
, dw_probe
);
1452 module_init(dw_init
);
1454 static void __exit
dw_exit(void)
1456 platform_driver_unregister(&dw_driver
);
1458 module_exit(dw_exit
);
1460 MODULE_LICENSE("GPL v2");
1461 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1462 MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");