Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com> | |
9a88cbb5 | 7 | * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org> |
1da177e4 LT |
8 | * swiped from i386, and cloned for MIPS by Geert, polished by Ralf. |
9 | */ | |
9a88cbb5 | 10 | |
1da177e4 | 11 | #include <linux/types.h> |
9a88cbb5 | 12 | #include <linux/dma-mapping.h> |
1da177e4 LT |
13 | #include <linux/mm.h> |
14 | #include <linux/module.h> | |
4fcc47a0 | 15 | #include <linux/scatterlist.h> |
6e86b0bf | 16 | #include <linux/string.h> |
5a0e3ad6 | 17 | #include <linux/gfp.h> |
1da177e4 LT |
18 | |
19 | #include <asm/cache.h> | |
20 | #include <asm/io.h> | |
21 | ||
9a88cbb5 RB |
22 | #include <dma-coherence.h> |
23 | ||
3807ef3f KC |
24 | static inline unsigned long dma_addr_to_virt(struct device *dev, |
25 | dma_addr_t dma_addr) | |
c9d06962 | 26 | { |
3807ef3f | 27 | unsigned long addr = plat_dma_addr_to_phys(dev, dma_addr); |
c9d06962 FBH |
28 | |
29 | return (unsigned long)phys_to_virt(addr); | |
30 | } | |
31 | ||
1da177e4 LT |
32 | /* |
33 | * Warning on the terminology - Linux calls an uncached area coherent; | |
34 | * MIPS terminology calls memory areas with hardware maintained coherency | |
35 | * coherent. | |
36 | */ | |
37 | ||
9a88cbb5 RB |
38 | static inline int cpu_is_noncoherent_r10000(struct device *dev) |
39 | { | |
40 | return !plat_device_is_coherent(dev) && | |
10cc3529 RB |
41 | (current_cpu_type() == CPU_R10000 || |
42 | current_cpu_type() == CPU_R12000); | |
9a88cbb5 RB |
43 | } |
44 | ||
cce335ae RB |
45 | static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp) |
46 | { | |
a2e715a8 RB |
47 | gfp_t dma_flag; |
48 | ||
cce335ae RB |
49 | /* ignore region specifiers */ |
50 | gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); | |
51 | ||
a2e715a8 | 52 | #ifdef CONFIG_ISA |
cce335ae | 53 | if (dev == NULL) |
a2e715a8 | 54 | dma_flag = __GFP_DMA; |
cce335ae RB |
55 | else |
56 | #endif | |
a2e715a8 | 57 | #if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA) |
cce335ae | 58 | if (dev->coherent_dma_mask < DMA_BIT_MASK(32)) |
a2e715a8 RB |
59 | dma_flag = __GFP_DMA; |
60 | else if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) | |
61 | dma_flag = __GFP_DMA32; | |
62 | else | |
63 | #endif | |
64 | #if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA) | |
65 | if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) | |
66 | dma_flag = __GFP_DMA32; | |
67 | else | |
68 | #endif | |
69 | #if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32) | |
70 | if (dev->coherent_dma_mask < DMA_BIT_MASK(64)) | |
71 | dma_flag = __GFP_DMA; | |
cce335ae RB |
72 | else |
73 | #endif | |
a2e715a8 | 74 | dma_flag = 0; |
cce335ae RB |
75 | |
76 | /* Don't invoke OOM killer */ | |
77 | gfp |= __GFP_NORETRY; | |
78 | ||
a2e715a8 | 79 | return gfp | dma_flag; |
cce335ae RB |
80 | } |
81 | ||
1da177e4 | 82 | void *dma_alloc_noncoherent(struct device *dev, size_t size, |
185a8ff5 | 83 | dma_addr_t * dma_handle, gfp_t gfp) |
1da177e4 LT |
84 | { |
85 | void *ret; | |
9a88cbb5 | 86 | |
cce335ae | 87 | gfp = massage_gfp_flags(dev, gfp); |
1da177e4 | 88 | |
1da177e4 LT |
89 | ret = (void *) __get_free_pages(gfp, get_order(size)); |
90 | ||
91 | if (ret != NULL) { | |
92 | memset(ret, 0, size); | |
9a88cbb5 | 93 | *dma_handle = plat_map_dma_mem(dev, ret, size); |
1da177e4 LT |
94 | } |
95 | ||
96 | return ret; | |
97 | } | |
1da177e4 LT |
98 | EXPORT_SYMBOL(dma_alloc_noncoherent); |
99 | ||
48e1fd5a | 100 | static void *mips_dma_alloc_coherent(struct device *dev, size_t size, |
185a8ff5 | 101 | dma_addr_t * dma_handle, gfp_t gfp) |
1da177e4 LT |
102 | { |
103 | void *ret; | |
104 | ||
f8ac0425 YY |
105 | if (dma_alloc_from_coherent(dev, size, dma_handle, &ret)) |
106 | return ret; | |
107 | ||
cce335ae | 108 | gfp = massage_gfp_flags(dev, gfp); |
9a88cbb5 | 109 | |
9a88cbb5 RB |
110 | ret = (void *) __get_free_pages(gfp, get_order(size)); |
111 | ||
1da177e4 | 112 | if (ret) { |
9a88cbb5 RB |
113 | memset(ret, 0, size); |
114 | *dma_handle = plat_map_dma_mem(dev, ret, size); | |
115 | ||
116 | if (!plat_device_is_coherent(dev)) { | |
117 | dma_cache_wback_inv((unsigned long) ret, size); | |
118 | ret = UNCAC_ADDR(ret); | |
119 | } | |
1da177e4 LT |
120 | } |
121 | ||
122 | return ret; | |
123 | } | |
124 | ||
1da177e4 LT |
125 | |
126 | void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr, | |
127 | dma_addr_t dma_handle) | |
128 | { | |
d3f634b9 | 129 | plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); |
1da177e4 LT |
130 | free_pages((unsigned long) vaddr, get_order(size)); |
131 | } | |
1da177e4 LT |
132 | EXPORT_SYMBOL(dma_free_noncoherent); |
133 | ||
48e1fd5a | 134 | static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, |
1da177e4 LT |
135 | dma_addr_t dma_handle) |
136 | { | |
137 | unsigned long addr = (unsigned long) vaddr; | |
f8ac0425 YY |
138 | int order = get_order(size); |
139 | ||
140 | if (dma_release_from_coherent(dev, order, vaddr)) | |
141 | return; | |
1da177e4 | 142 | |
d3f634b9 | 143 | plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); |
11531ac2 | 144 | |
9a88cbb5 RB |
145 | if (!plat_device_is_coherent(dev)) |
146 | addr = CAC_ADDR(addr); | |
147 | ||
1da177e4 LT |
148 | free_pages(addr, get_order(size)); |
149 | } | |
150 | ||
1da177e4 LT |
151 | static inline void __dma_sync(unsigned long addr, size_t size, |
152 | enum dma_data_direction direction) | |
153 | { | |
154 | switch (direction) { | |
155 | case DMA_TO_DEVICE: | |
156 | dma_cache_wback(addr, size); | |
157 | break; | |
158 | ||
159 | case DMA_FROM_DEVICE: | |
160 | dma_cache_inv(addr, size); | |
161 | break; | |
162 | ||
163 | case DMA_BIDIRECTIONAL: | |
164 | dma_cache_wback_inv(addr, size); | |
165 | break; | |
166 | ||
167 | default: | |
168 | BUG(); | |
169 | } | |
170 | } | |
171 | ||
48e1fd5a DD |
172 | static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, |
173 | size_t size, enum dma_data_direction direction, struct dma_attrs *attrs) | |
1da177e4 | 174 | { |
9a88cbb5 | 175 | if (cpu_is_noncoherent_r10000(dev)) |
3807ef3f | 176 | __dma_sync(dma_addr_to_virt(dev, dma_addr), size, |
9a88cbb5 | 177 | direction); |
1da177e4 | 178 | |
d3f634b9 | 179 | plat_unmap_dma_mem(dev, dma_addr, size, direction); |
1da177e4 LT |
180 | } |
181 | ||
48e1fd5a DD |
182 | static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg, |
183 | int nents, enum dma_data_direction direction, struct dma_attrs *attrs) | |
1da177e4 LT |
184 | { |
185 | int i; | |
186 | ||
1da177e4 LT |
187 | for (i = 0; i < nents; i++, sg++) { |
188 | unsigned long addr; | |
42a3b4f2 | 189 | |
58b053e4 | 190 | addr = (unsigned long) sg_virt(sg); |
9a88cbb5 | 191 | if (!plat_device_is_coherent(dev) && addr) |
58b053e4 | 192 | __dma_sync(addr, sg->length, direction); |
fbd5604d | 193 | sg->dma_address = plat_map_dma_mem(dev, |
58b053e4 | 194 | (void *)addr, sg->length); |
1da177e4 LT |
195 | } |
196 | ||
197 | return nents; | |
198 | } | |
199 | ||
48e1fd5a DD |
200 | static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page, |
201 | unsigned long offset, size_t size, enum dma_data_direction direction, | |
202 | struct dma_attrs *attrs) | |
1da177e4 | 203 | { |
48e1fd5a | 204 | unsigned long addr; |
1da177e4 | 205 | |
48e1fd5a | 206 | addr = (unsigned long) page_address(page) + offset; |
9a88cbb5 | 207 | |
48e1fd5a | 208 | if (!plat_device_is_coherent(dev)) |
4f29c057 | 209 | __dma_sync(addr, size, direction); |
1da177e4 | 210 | |
48e1fd5a | 211 | return plat_map_dma_mem(dev, (void *)addr, size); |
1da177e4 LT |
212 | } |
213 | ||
48e1fd5a DD |
214 | static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg, |
215 | int nhwentries, enum dma_data_direction direction, | |
216 | struct dma_attrs *attrs) | |
1da177e4 LT |
217 | { |
218 | unsigned long addr; | |
219 | int i; | |
220 | ||
1da177e4 | 221 | for (i = 0; i < nhwentries; i++, sg++) { |
9a88cbb5 RB |
222 | if (!plat_device_is_coherent(dev) && |
223 | direction != DMA_TO_DEVICE) { | |
58b053e4 | 224 | addr = (unsigned long) sg_virt(sg); |
9a88cbb5 | 225 | if (addr) |
58b053e4 | 226 | __dma_sync(addr, sg->length, direction); |
9a88cbb5 | 227 | } |
d3f634b9 | 228 | plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction); |
1da177e4 LT |
229 | } |
230 | } | |
231 | ||
48e1fd5a DD |
232 | static void mips_dma_sync_single_for_cpu(struct device *dev, |
233 | dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) | |
1da177e4 | 234 | { |
9a88cbb5 RB |
235 | if (cpu_is_noncoherent_r10000(dev)) { |
236 | unsigned long addr; | |
237 | ||
3807ef3f | 238 | addr = dma_addr_to_virt(dev, dma_handle); |
9a88cbb5 RB |
239 | __dma_sync(addr, size, direction); |
240 | } | |
1da177e4 LT |
241 | } |
242 | ||
48e1fd5a DD |
243 | static void mips_dma_sync_single_for_device(struct device *dev, |
244 | dma_addr_t dma_handle, size_t size, enum dma_data_direction direction) | |
1da177e4 | 245 | { |
843aef49 | 246 | plat_extra_sync_for_device(dev); |
9b43fb6b | 247 | if (!plat_device_is_coherent(dev)) { |
9a88cbb5 RB |
248 | unsigned long addr; |
249 | ||
3807ef3f | 250 | addr = dma_addr_to_virt(dev, dma_handle); |
9a88cbb5 RB |
251 | __dma_sync(addr, size, direction); |
252 | } | |
1da177e4 LT |
253 | } |
254 | ||
48e1fd5a DD |
255 | static void mips_dma_sync_sg_for_cpu(struct device *dev, |
256 | struct scatterlist *sg, int nelems, enum dma_data_direction direction) | |
1da177e4 LT |
257 | { |
258 | int i; | |
42a3b4f2 | 259 | |
1da177e4 | 260 | /* Make sure that gcc doesn't leave the empty loop body. */ |
9a88cbb5 | 261 | for (i = 0; i < nelems; i++, sg++) { |
5b648a98 | 262 | if (cpu_is_noncoherent_r10000(dev)) |
58b053e4 | 263 | __dma_sync((unsigned long)page_address(sg_page(sg)), |
9a88cbb5 | 264 | sg->length, direction); |
9a88cbb5 | 265 | } |
1da177e4 LT |
266 | } |
267 | ||
48e1fd5a DD |
268 | static void mips_dma_sync_sg_for_device(struct device *dev, |
269 | struct scatterlist *sg, int nelems, enum dma_data_direction direction) | |
1da177e4 LT |
270 | { |
271 | int i; | |
272 | ||
1da177e4 | 273 | /* Make sure that gcc doesn't leave the empty loop body. */ |
9a88cbb5 RB |
274 | for (i = 0; i < nelems; i++, sg++) { |
275 | if (!plat_device_is_coherent(dev)) | |
58b053e4 | 276 | __dma_sync((unsigned long)page_address(sg_page(sg)), |
9a88cbb5 | 277 | sg->length, direction); |
9a88cbb5 | 278 | } |
1da177e4 LT |
279 | } |
280 | ||
48e1fd5a | 281 | int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) |
1da177e4 | 282 | { |
843aef49 | 283 | return plat_dma_mapping_error(dev, dma_addr); |
1da177e4 LT |
284 | } |
285 | ||
48e1fd5a | 286 | int mips_dma_supported(struct device *dev, u64 mask) |
1da177e4 | 287 | { |
843aef49 | 288 | return plat_dma_supported(dev, mask); |
1da177e4 LT |
289 | } |
290 | ||
48e1fd5a DD |
291 | void mips_dma_cache_sync(struct device *dev, void *vaddr, size_t size, |
292 | enum dma_data_direction direction) | |
1da177e4 | 293 | { |
9a88cbb5 | 294 | BUG_ON(direction == DMA_NONE); |
1da177e4 | 295 | |
843aef49 | 296 | plat_extra_sync_for_device(dev); |
9a88cbb5 | 297 | if (!plat_device_is_coherent(dev)) |
c7c6b390 | 298 | __dma_sync((unsigned long)vaddr, size, direction); |
1da177e4 LT |
299 | } |
300 | ||
48e1fd5a DD |
301 | static struct dma_map_ops mips_default_dma_map_ops = { |
302 | .alloc_coherent = mips_dma_alloc_coherent, | |
303 | .free_coherent = mips_dma_free_coherent, | |
304 | .map_page = mips_dma_map_page, | |
305 | .unmap_page = mips_dma_unmap_page, | |
306 | .map_sg = mips_dma_map_sg, | |
307 | .unmap_sg = mips_dma_unmap_sg, | |
308 | .sync_single_for_cpu = mips_dma_sync_single_for_cpu, | |
309 | .sync_single_for_device = mips_dma_sync_single_for_device, | |
310 | .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu, | |
311 | .sync_sg_for_device = mips_dma_sync_sg_for_device, | |
312 | .mapping_error = mips_dma_mapping_error, | |
313 | .dma_supported = mips_dma_supported | |
314 | }; | |
315 | ||
316 | struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops; | |
317 | EXPORT_SYMBOL(mips_dma_map_ops); | |
318 | ||
319 | #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) | |
320 | ||
321 | static int __init mips_dma_init(void) | |
322 | { | |
323 | dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); | |
324 | ||
325 | return 0; | |
326 | } | |
327 | fs_initcall(mips_dma_init); |