Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Dynamic DMA mapping support. | |
3 | */ | |
4 | ||
5 | #include <linux/types.h> | |
6 | #include <linux/mm.h> | |
7 | #include <linux/string.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/module.h> | |
10 | #include <asm/io.h> | |
17a941d8 | 11 | #include <asm/proto.h> |
1da177e4 | 12 | |
17a941d8 MBY |
13 | int iommu_merge __read_mostly = 0; |
14 | EXPORT_SYMBOL(iommu_merge); | |
15 | ||
16 | dma_addr_t bad_dma_address __read_mostly; | |
17 | EXPORT_SYMBOL(bad_dma_address); | |
18 | ||
19 | /* This tells the BIO block layer to assume merging. Default to off | |
20 | because we cannot guarantee merging later. */ | |
21 | int iommu_bio_merge __read_mostly = 0; | |
22 | EXPORT_SYMBOL(iommu_bio_merge); | |
23 | ||
24 | int iommu_sac_force __read_mostly = 0; | |
25 | EXPORT_SYMBOL(iommu_sac_force); | |
26 | ||
27 | int no_iommu __read_mostly; | |
28 | #ifdef CONFIG_IOMMU_DEBUG | |
29 | int panic_on_overflow __read_mostly = 1; | |
30 | int force_iommu __read_mostly = 1; | |
31 | #else | |
32 | int panic_on_overflow __read_mostly = 0; | |
33 | int force_iommu __read_mostly= 0; | |
34 | #endif | |
35 | ||
36 | /* Dummy device used for NULL arguments (normally ISA). Better would | |
37 | be probably a smaller DMA mask, but this is bug-to-bug compatible | |
38 | to i386. */ | |
39 | struct device fallback_dev = { | |
40 | .bus_id = "fallback device", | |
41 | .coherent_dma_mask = 0xffffffff, | |
42 | .dma_mask = &fallback_dev.coherent_dma_mask, | |
43 | }; | |
44 | ||
45 | /* Allocate DMA memory on node near device */ | |
46 | noinline static void * | |
47 | dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order) | |
48 | { | |
49 | struct page *page; | |
50 | int node; | |
51 | if (dev->bus == &pci_bus_type) | |
52 | node = pcibus_to_node(to_pci_dev(dev)->bus); | |
53 | else | |
54 | node = numa_node_id(); | |
55 | page = alloc_pages_node(node, gfp, order); | |
56 | return page ? page_address(page) : NULL; | |
57 | } | |
58 | ||
59 | /* | |
60 | * Allocate memory for a coherent mapping. | |
1da177e4 | 61 | */ |
17a941d8 MBY |
62 | void * |
63 | dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, | |
64 | gfp_t gfp) | |
1da177e4 | 65 | { |
17a941d8 MBY |
66 | void *memory; |
67 | unsigned long dma_mask = 0; | |
68 | u64 bus; | |
69 | ||
70 | if (!dev) | |
71 | dev = &fallback_dev; | |
72 | dma_mask = dev->coherent_dma_mask; | |
73 | if (dma_mask == 0) | |
74 | dma_mask = 0xffffffff; | |
75 | ||
76 | /* Kludge to make it bug-to-bug compatible with i386. i386 | |
77 | uses the normal dma_mask for alloc_coherent. */ | |
78 | dma_mask &= *dev->dma_mask; | |
79 | ||
80 | /* Why <=? Even when the mask is smaller than 4GB it is often | |
81 | larger than 16MB and in this case we have a chance of | |
82 | finding fitting memory in the next higher zone first. If | |
83 | not retry with true GFP_DMA. -AK */ | |
84 | if (dma_mask <= 0xffffffff) | |
85 | gfp |= GFP_DMA32; | |
86 | ||
87 | again: | |
88 | memory = dma_alloc_pages(dev, gfp, get_order(size)); | |
89 | if (memory == NULL) | |
90 | return NULL; | |
91 | ||
92 | { | |
93 | int high, mmu; | |
94 | bus = virt_to_bus(memory); | |
95 | high = (bus + size) >= dma_mask; | |
96 | mmu = high; | |
97 | if (force_iommu && !(gfp & GFP_DMA)) | |
98 | mmu = 1; | |
99 | else if (high) { | |
100 | free_pages((unsigned long)memory, | |
101 | get_order(size)); | |
102 | ||
103 | /* Don't use the 16MB ZONE_DMA unless absolutely | |
104 | needed. It's better to use remapping first. */ | |
105 | if (dma_mask < 0xffffffff && !(gfp & GFP_DMA)) { | |
106 | gfp = (gfp & ~GFP_DMA32) | GFP_DMA; | |
107 | goto again; | |
108 | } | |
109 | ||
6bca52b5 AK |
110 | /* Let low level make its own zone decisions */ |
111 | gfp &= ~(GFP_DMA32|GFP_DMA); | |
112 | ||
17a941d8 MBY |
113 | if (dma_ops->alloc_coherent) |
114 | return dma_ops->alloc_coherent(dev, size, | |
115 | dma_handle, gfp); | |
116 | return NULL; | |
117 | } | |
118 | ||
119 | memset(memory, 0, size); | |
120 | if (!mmu) { | |
121 | *dma_handle = virt_to_bus(memory); | |
122 | return memory; | |
123 | } | |
124 | } | |
125 | ||
126 | if (dma_ops->alloc_coherent) { | |
127 | free_pages((unsigned long)memory, get_order(size)); | |
128 | gfp &= ~(GFP_DMA|GFP_DMA32); | |
129 | return dma_ops->alloc_coherent(dev, size, dma_handle, gfp); | |
130 | } | |
131 | ||
132 | if (dma_ops->map_simple) { | |
133 | *dma_handle = dma_ops->map_simple(dev, memory, | |
134 | size, | |
135 | PCI_DMA_BIDIRECTIONAL); | |
136 | if (*dma_handle != bad_dma_address) | |
137 | return memory; | |
1da177e4 | 138 | } |
1da177e4 | 139 | |
17a941d8 MBY |
140 | if (panic_on_overflow) |
141 | panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size); | |
142 | free_pages((unsigned long)memory, get_order(size)); | |
143 | return NULL; | |
144 | } | |
145 | EXPORT_SYMBOL(dma_alloc_coherent); | |
1da177e4 | 146 | |
17a941d8 MBY |
147 | /* |
148 | * Unmap coherent memory. | |
149 | * The caller must ensure that the device has finished accessing the mapping. | |
1da177e4 | 150 | */ |
17a941d8 MBY |
151 | void dma_free_coherent(struct device *dev, size_t size, |
152 | void *vaddr, dma_addr_t bus) | |
153 | { | |
154 | if (dma_ops->unmap_single) | |
155 | dma_ops->unmap_single(dev, bus, size, 0); | |
156 | free_pages((unsigned long)vaddr, get_order(size)); | |
157 | } | |
158 | EXPORT_SYMBOL(dma_free_coherent); | |
159 | ||
160 | int dma_supported(struct device *dev, u64 mask) | |
161 | { | |
162 | if (dma_ops->dma_supported) | |
163 | return dma_ops->dma_supported(dev, mask); | |
164 | ||
165 | /* Copied from i386. Doesn't make much sense, because it will | |
166 | only work for pci_alloc_coherent. | |
167 | The caller just has to use GFP_DMA in this case. */ | |
168 | if (mask < 0x00ffffff) | |
169 | return 0; | |
170 | ||
171 | /* Tell the device to use SAC when IOMMU force is on. This | |
172 | allows the driver to use cheaper accesses in some cases. | |
173 | ||
174 | Problem with this is that if we overflow the IOMMU area and | |
175 | return DAC as fallback address the device may not handle it | |
176 | correctly. | |
177 | ||
178 | As a special case some controllers have a 39bit address | |
179 | mode that is as efficient as 32bit (aic79xx). Don't force | |
180 | SAC for these. Assume all masks <= 40 bits are of this | |
181 | type. Normally this doesn't make any difference, but gives | |
182 | more gentle handling of IOMMU overflow. */ | |
183 | if (iommu_sac_force && (mask >= 0xffffffffffULL)) { | |
184 | printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask); | |
185 | return 0; | |
186 | } | |
187 | ||
188 | return 1; | |
189 | } | |
190 | EXPORT_SYMBOL(dma_supported); | |
191 | ||
192 | int dma_set_mask(struct device *dev, u64 mask) | |
1da177e4 | 193 | { |
17a941d8 MBY |
194 | if (!dev->dma_mask || !dma_supported(dev, mask)) |
195 | return -EIO; | |
196 | *dev->dma_mask = mask; | |
197 | return 0; | |
1da177e4 | 198 | } |
17a941d8 MBY |
199 | EXPORT_SYMBOL(dma_set_mask); |
200 | ||
201 | /* iommu=[size][,noagp][,off][,force][,noforce][,leak][,memaper[=order]][,merge] | |
202 | [,forcesac][,fullflush][,nomerge][,biomerge] | |
203 | size set size of iommu (in bytes) | |
204 | noagp don't initialize the AGP driver and use full aperture. | |
205 | off don't use the IOMMU | |
206 | leak turn on simple iommu leak tracing (only when CONFIG_IOMMU_LEAK is on) | |
207 | memaper[=order] allocate an own aperture over RAM with size 32MB^order. | |
208 | noforce don't force IOMMU usage. Default. | |
209 | force Force IOMMU. | |
210 | merge Do lazy merging. This may improve performance on some block devices. | |
211 | Implies force (experimental) | |
212 | biomerge Do merging at the BIO layer. This is more efficient than merge, | |
213 | but should be only done with very big IOMMUs. Implies merge,force. | |
214 | nomerge Don't do SG merging. | |
215 | forcesac For SAC mode for masks <40bits (experimental) | |
216 | fullflush Flush IOMMU on each allocation (default) | |
217 | nofullflush Don't use IOMMU fullflush | |
218 | allowed overwrite iommu off workarounds for specific chipsets. | |
219 | soft Use software bounce buffering (default for Intel machines) | |
220 | noaperture Don't touch the aperture for AGP. | |
221 | */ | |
222 | __init int iommu_setup(char *p) | |
223 | { | |
224 | iommu_merge = 1; | |
1da177e4 | 225 | |
17a941d8 MBY |
226 | while (*p) { |
227 | if (!strncmp(p,"off",3)) | |
228 | no_iommu = 1; | |
229 | /* gart_parse_options has more force support */ | |
230 | if (!strncmp(p,"force",5)) | |
231 | force_iommu = 1; | |
232 | if (!strncmp(p,"noforce",7)) { | |
233 | iommu_merge = 0; | |
234 | force_iommu = 0; | |
235 | } | |
236 | ||
237 | if (!strncmp(p, "biomerge",8)) { | |
238 | iommu_bio_merge = 4096; | |
239 | iommu_merge = 1; | |
240 | force_iommu = 1; | |
241 | } | |
242 | if (!strncmp(p, "panic",5)) | |
243 | panic_on_overflow = 1; | |
244 | if (!strncmp(p, "nopanic",7)) | |
245 | panic_on_overflow = 0; | |
246 | if (!strncmp(p, "merge",5)) { | |
247 | iommu_merge = 1; | |
248 | force_iommu = 1; | |
249 | } | |
250 | if (!strncmp(p, "nomerge",7)) | |
251 | iommu_merge = 0; | |
252 | if (!strncmp(p, "forcesac",8)) | |
253 | iommu_sac_force = 1; | |
254 | ||
255 | #ifdef CONFIG_SWIOTLB | |
256 | if (!strncmp(p, "soft",4)) | |
257 | swiotlb = 1; | |
258 | #endif | |
259 | ||
260 | #ifdef CONFIG_GART_IOMMU | |
261 | gart_parse_options(p); | |
262 | #endif | |
263 | ||
264 | p += strcspn(p, ","); | |
265 | if (*p == ',') | |
266 | ++p; | |
267 | } | |
268 | return 1; | |
269 | } |