Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Nvidia AGPGART routines. | |
3 | * Based upon a 2.4 agpgart diff by the folks from NVIDIA, and hacked up | |
f4432c5c | 4 | * to work in 2.5 by Dave Jones <davej@redhat.com> |
1da177e4 LT |
5 | */ |
6 | ||
7 | #include <linux/module.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/agp_backend.h> | |
11 | #include <linux/gfp.h> | |
12 | #include <linux/page-flags.h> | |
13 | #include <linux/mm.h> | |
05613bdd | 14 | #include <linux/jiffies.h> |
1da177e4 LT |
15 | #include "agp.h" |
16 | ||
17 | /* NVIDIA registers */ | |
18 | #define NVIDIA_0_APSIZE 0x80 | |
19 | #define NVIDIA_1_WBC 0xf0 | |
20 | #define NVIDIA_2_GARTCTRL 0xd0 | |
21 | #define NVIDIA_2_APBASE 0xd8 | |
22 | #define NVIDIA_2_APLIMIT 0xdc | |
23 | #define NVIDIA_2_ATTBASE(i) (0xe0 + (i) * 4) | |
24 | #define NVIDIA_3_APBASE 0x50 | |
25 | #define NVIDIA_3_APLIMIT 0x54 | |
26 | ||
27 | ||
28 | static struct _nvidia_private { | |
29 | struct pci_dev *dev_1; | |
30 | struct pci_dev *dev_2; | |
31 | struct pci_dev *dev_3; | |
32 | volatile u32 __iomem *aperture; | |
33 | int num_active_entries; | |
34 | off_t pg_offset; | |
35 | u32 wbc_mask; | |
36 | } nvidia_private; | |
37 | ||
38 | ||
39 | static int nvidia_fetch_size(void) | |
40 | { | |
41 | int i; | |
42 | u8 size_value; | |
43 | struct aper_size_info_8 *values; | |
44 | ||
45 | pci_read_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, &size_value); | |
46 | size_value &= 0x0f; | |
47 | values = A_SIZE_8(agp_bridge->driver->aperture_sizes); | |
48 | ||
49 | for (i = 0; i < agp_bridge->driver->num_aperture_sizes; i++) { | |
50 | if (size_value == values[i].size_value) { | |
51 | agp_bridge->previous_size = | |
52 | agp_bridge->current_size = (void *) (values + i); | |
53 | agp_bridge->aperture_size_idx = i; | |
54 | return values[i].size; | |
55 | } | |
56 | } | |
57 | ||
58 | return 0; | |
59 | } | |
60 | ||
61 | #define SYSCFG 0xC0010010 | |
62 | #define IORR_BASE0 0xC0010016 | |
63 | #define IORR_MASK0 0xC0010017 | |
64 | #define AMD_K7_NUM_IORR 2 | |
65 | ||
66 | static int nvidia_init_iorr(u32 base, u32 size) | |
67 | { | |
68 | u32 base_hi, base_lo; | |
69 | u32 mask_hi, mask_lo; | |
70 | u32 sys_hi, sys_lo; | |
71 | u32 iorr_addr, free_iorr_addr; | |
72 | ||
73 | /* Find the iorr that is already used for the base */ | |
74 | /* If not found, determine the uppermost available iorr */ | |
75 | free_iorr_addr = AMD_K7_NUM_IORR; | |
6a92a4e0 | 76 | for (iorr_addr = 0; iorr_addr < AMD_K7_NUM_IORR; iorr_addr++) { |
1da177e4 LT |
77 | rdmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi); |
78 | rdmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi); | |
79 | ||
80 | if ((base_lo & 0xfffff000) == (base & 0xfffff000)) | |
81 | break; | |
82 | ||
83 | if ((mask_lo & 0x00000800) == 0) | |
84 | free_iorr_addr = iorr_addr; | |
85 | } | |
6a92a4e0 | 86 | |
1da177e4 LT |
87 | if (iorr_addr >= AMD_K7_NUM_IORR) { |
88 | iorr_addr = free_iorr_addr; | |
89 | if (iorr_addr >= AMD_K7_NUM_IORR) | |
90 | return -EINVAL; | |
91 | } | |
92 | base_hi = 0x0; | |
93 | base_lo = (base & ~0xfff) | 0x18; | |
94 | mask_hi = 0xf; | |
95 | mask_lo = ((~(size - 1)) & 0xfffff000) | 0x800; | |
96 | wrmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi); | |
97 | wrmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi); | |
98 | ||
99 | rdmsr(SYSCFG, sys_lo, sys_hi); | |
100 | sys_lo |= 0x00100000; | |
101 | wrmsr(SYSCFG, sys_lo, sys_hi); | |
102 | ||
103 | return 0; | |
104 | } | |
105 | ||
106 | static int nvidia_configure(void) | |
107 | { | |
108 | int i, rc, num_dirs; | |
109 | u32 apbase, aplimit; | |
110 | struct aper_size_info_8 *current_size; | |
111 | u32 temp; | |
112 | ||
113 | current_size = A_SIZE_8(agp_bridge->current_size); | |
114 | ||
115 | /* aperture size */ | |
116 | pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, | |
117 | current_size->size_value); | |
118 | ||
119 | /* address to map to */ | |
120 | pci_read_config_dword(agp_bridge->dev, AGP_APBASE, &apbase); | |
121 | apbase &= PCI_BASE_ADDRESS_MEM_MASK; | |
122 | agp_bridge->gart_bus_addr = apbase; | |
123 | aplimit = apbase + (current_size->size * 1024 * 1024) - 1; | |
124 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APBASE, apbase); | |
125 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APLIMIT, aplimit); | |
126 | pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APBASE, apbase); | |
127 | pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APLIMIT, aplimit); | |
128 | if (0 != (rc = nvidia_init_iorr(apbase, current_size->size * 1024 * 1024))) | |
129 | return rc; | |
130 | ||
131 | /* directory size is 64k */ | |
132 | num_dirs = current_size->size / 64; | |
133 | nvidia_private.num_active_entries = current_size->num_entries; | |
134 | nvidia_private.pg_offset = 0; | |
135 | if (num_dirs == 0) { | |
136 | num_dirs = 1; | |
137 | nvidia_private.num_active_entries /= (64 / current_size->size); | |
138 | nvidia_private.pg_offset = (apbase & (64 * 1024 * 1024 - 1) & | |
139 | ~(current_size->size * 1024 * 1024 - 1)) / PAGE_SIZE; | |
140 | } | |
141 | ||
142 | /* attbase */ | |
6a92a4e0 | 143 | for (i = 0; i < 8; i++) { |
1da177e4 LT |
144 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_ATTBASE(i), |
145 | (agp_bridge->gatt_bus_addr + (i % num_dirs) * 64 * 1024) | 1); | |
146 | } | |
147 | ||
148 | /* gtlb control */ | |
149 | pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp); | |
150 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp | 0x11); | |
151 | ||
152 | /* gart control */ | |
153 | pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp); | |
154 | pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp | 0x100); | |
155 | ||
156 | /* map aperture */ | |
157 | nvidia_private.aperture = | |
158 | (volatile u32 __iomem *) ioremap(apbase, 33 * PAGE_SIZE); | |
159 | ||
5bdbc7dc ST |
160 | if (!nvidia_private.aperture) |
161 | return -ENOMEM; | |
162 | ||
1da177e4 LT |
163 | return 0; |
164 | } | |
165 | ||
166 | static void nvidia_cleanup(void) | |
167 | { | |
168 | struct aper_size_info_8 *previous_size; | |
169 | u32 temp; | |
170 | ||
171 | /* gart control */ | |
172 | pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp); | |
173 | pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp & ~(0x100)); | |
174 | ||
175 | /* gtlb control */ | |
176 | pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp); | |
177 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp & ~(0x11)); | |
178 | ||
179 | /* unmap aperture */ | |
180 | iounmap((void __iomem *) nvidia_private.aperture); | |
181 | ||
182 | /* restore previous aperture size */ | |
183 | previous_size = A_SIZE_8(agp_bridge->previous_size); | |
184 | pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, | |
185 | previous_size->size_value); | |
186 | ||
187 | /* restore iorr for previous aperture size */ | |
188 | nvidia_init_iorr(agp_bridge->gart_bus_addr, | |
189 | previous_size->size * 1024 * 1024); | |
190 | } | |
191 | ||
192 | ||
193 | /* | |
194 | * Note we can't use the generic routines, even though they are 99% the same. | |
195 | * Aperture sizes <64M still requires a full 64k GART directory, but | |
196 | * only use the portion of the TLB entries that correspond to the apertures | |
197 | * alignment inside the surrounding 64M block. | |
198 | */ | |
199 | extern int agp_memory_reserved; | |
200 | ||
201 | static int nvidia_insert_memory(struct agp_memory *mem, off_t pg_start, int type) | |
202 | { | |
203 | int i, j; | |
a64d2b37 | 204 | int mask_type; |
6a92a4e0 | 205 | |
a64d2b37 TH |
206 | mask_type = agp_generic_type_to_mask_type(mem->bridge, type); |
207 | if (mask_type != 0 || type != mem->type) | |
1da177e4 | 208 | return -EINVAL; |
6a92a4e0 | 209 | |
a64d2b37 TH |
210 | if (mem->page_count == 0) |
211 | return 0; | |
212 | ||
1da177e4 LT |
213 | if ((pg_start + mem->page_count) > |
214 | (nvidia_private.num_active_entries - agp_memory_reserved/PAGE_SIZE)) | |
215 | return -EINVAL; | |
6a92a4e0 DJ |
216 | |
217 | for (j = pg_start; j < (pg_start + mem->page_count); j++) { | |
1da177e4 LT |
218 | if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j))) |
219 | return -EBUSY; | |
220 | } | |
221 | ||
c7258012 | 222 | if (!mem->is_flushed) { |
1da177e4 | 223 | global_cache_flush(); |
c7258012 | 224 | mem->is_flushed = true; |
1da177e4 LT |
225 | } |
226 | for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { | |
227 | writel(agp_bridge->driver->mask_memory(agp_bridge, | |
07613ba2 | 228 | mem->pages[i], mask_type), |
1da177e4 | 229 | agp_bridge->gatt_table+nvidia_private.pg_offset+j); |
1da177e4 | 230 | } |
a64d2b37 TH |
231 | |
232 | /* PCI Posting. */ | |
233 | readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j - 1); | |
234 | ||
1da177e4 LT |
235 | agp_bridge->driver->tlb_flush(mem); |
236 | return 0; | |
237 | } | |
238 | ||
239 | ||
240 | static int nvidia_remove_memory(struct agp_memory *mem, off_t pg_start, int type) | |
241 | { | |
242 | int i; | |
243 | ||
a64d2b37 TH |
244 | int mask_type; |
245 | ||
246 | mask_type = agp_generic_type_to_mask_type(mem->bridge, type); | |
247 | if (mask_type != 0 || type != mem->type) | |
1da177e4 LT |
248 | return -EINVAL; |
249 | ||
a64d2b37 TH |
250 | if (mem->page_count == 0) |
251 | return 0; | |
252 | ||
1da177e4 LT |
253 | for (i = pg_start; i < (mem->page_count + pg_start); i++) |
254 | writel(agp_bridge->scratch_page, agp_bridge->gatt_table+nvidia_private.pg_offset+i); | |
255 | ||
256 | agp_bridge->driver->tlb_flush(mem); | |
257 | return 0; | |
258 | } | |
259 | ||
260 | ||
261 | static void nvidia_tlbflush(struct agp_memory *mem) | |
262 | { | |
263 | unsigned long end; | |
264 | u32 wbc_reg, temp; | |
265 | int i; | |
266 | ||
267 | /* flush chipset */ | |
268 | if (nvidia_private.wbc_mask) { | |
269 | pci_read_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, &wbc_reg); | |
270 | wbc_reg |= nvidia_private.wbc_mask; | |
271 | pci_write_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, wbc_reg); | |
272 | ||
273 | end = jiffies + 3*HZ; | |
274 | do { | |
275 | pci_read_config_dword(nvidia_private.dev_1, | |
276 | NVIDIA_1_WBC, &wbc_reg); | |
05613bdd | 277 | if (time_before_eq(end, jiffies)) { |
1da177e4 LT |
278 | printk(KERN_ERR PFX |
279 | "TLB flush took more than 3 seconds.\n"); | |
280 | } | |
281 | } while (wbc_reg & nvidia_private.wbc_mask); | |
282 | } | |
283 | ||
284 | /* flush TLB entries */ | |
6a92a4e0 | 285 | for (i = 0; i < 32 + 1; i++) |
1da177e4 | 286 | temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32))); |
6a92a4e0 | 287 | for (i = 0; i < 32 + 1; i++) |
1da177e4 LT |
288 | temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32))); |
289 | } | |
290 | ||
291 | ||
e5524f35 | 292 | static const struct aper_size_info_8 nvidia_generic_sizes[5] = |
1da177e4 LT |
293 | { |
294 | {512, 131072, 7, 0}, | |
295 | {256, 65536, 6, 8}, | |
296 | {128, 32768, 5, 12}, | |
297 | {64, 16384, 4, 14}, | |
298 | /* The 32M mode still requires a 64k gatt */ | |
299 | {32, 16384, 4, 15} | |
300 | }; | |
301 | ||
302 | ||
e5524f35 | 303 | static const struct gatt_mask nvidia_generic_masks[] = |
1da177e4 LT |
304 | { |
305 | { .mask = 1, .type = 0} | |
306 | }; | |
307 | ||
308 | ||
e5524f35 | 309 | static const struct agp_bridge_driver nvidia_driver = { |
1da177e4 LT |
310 | .owner = THIS_MODULE, |
311 | .aperture_sizes = nvidia_generic_sizes, | |
312 | .size_type = U8_APER_SIZE, | |
313 | .num_aperture_sizes = 5, | |
314 | .configure = nvidia_configure, | |
315 | .fetch_size = nvidia_fetch_size, | |
316 | .cleanup = nvidia_cleanup, | |
317 | .tlb_flush = nvidia_tlbflush, | |
318 | .mask_memory = agp_generic_mask_memory, | |
319 | .masks = nvidia_generic_masks, | |
320 | .agp_enable = agp_generic_enable, | |
321 | .cache_flush = global_cache_flush, | |
322 | .create_gatt_table = agp_generic_create_gatt_table, | |
323 | .free_gatt_table = agp_generic_free_gatt_table, | |
324 | .insert_memory = nvidia_insert_memory, | |
325 | .remove_memory = nvidia_remove_memory, | |
326 | .alloc_by_type = agp_generic_alloc_by_type, | |
327 | .free_by_type = agp_generic_free_by_type, | |
328 | .agp_alloc_page = agp_generic_alloc_page, | |
5f310b63 | 329 | .agp_alloc_pages = agp_generic_alloc_pages, |
1da177e4 | 330 | .agp_destroy_page = agp_generic_destroy_page, |
5f310b63 | 331 | .agp_destroy_pages = agp_generic_destroy_pages, |
a030ce44 | 332 | .agp_type_to_mask_type = agp_generic_type_to_mask_type, |
1da177e4 LT |
333 | }; |
334 | ||
335 | static int __devinit agp_nvidia_probe(struct pci_dev *pdev, | |
336 | const struct pci_device_id *ent) | |
337 | { | |
338 | struct agp_bridge_data *bridge; | |
339 | u8 cap_ptr; | |
340 | ||
341 | nvidia_private.dev_1 = | |
1e415732 | 342 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 1)); |
1da177e4 | 343 | nvidia_private.dev_2 = |
1e415732 | 344 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 2)); |
1da177e4 | 345 | nvidia_private.dev_3 = |
1e415732 | 346 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(30, 0)); |
6a92a4e0 | 347 | |
1da177e4 LT |
348 | if (!nvidia_private.dev_1 || !nvidia_private.dev_2 || !nvidia_private.dev_3) { |
349 | printk(KERN_INFO PFX "Detected an NVIDIA nForce/nForce2 " | |
350 | "chipset, but could not find the secondary devices.\n"); | |
351 | return -ENODEV; | |
352 | } | |
353 | ||
354 | cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP); | |
355 | if (!cap_ptr) | |
356 | return -ENODEV; | |
357 | ||
358 | switch (pdev->device) { | |
359 | case PCI_DEVICE_ID_NVIDIA_NFORCE: | |
360 | printk(KERN_INFO PFX "Detected NVIDIA nForce chipset\n"); | |
361 | nvidia_private.wbc_mask = 0x00010000; | |
362 | break; | |
363 | case PCI_DEVICE_ID_NVIDIA_NFORCE2: | |
364 | printk(KERN_INFO PFX "Detected NVIDIA nForce2 chipset\n"); | |
365 | nvidia_private.wbc_mask = 0x80000000; | |
366 | break; | |
367 | default: | |
368 | printk(KERN_ERR PFX "Unsupported NVIDIA chipset (device id: %04x)\n", | |
369 | pdev->device); | |
370 | return -ENODEV; | |
371 | } | |
372 | ||
373 | bridge = agp_alloc_bridge(); | |
374 | if (!bridge) | |
375 | return -ENOMEM; | |
376 | ||
377 | bridge->driver = &nvidia_driver; | |
378 | bridge->dev_private_data = &nvidia_private, | |
379 | bridge->dev = pdev; | |
380 | bridge->capndx = cap_ptr; | |
381 | ||
382 | /* Fill in the mode register */ | |
383 | pci_read_config_dword(pdev, | |
384 | bridge->capndx+PCI_AGP_STATUS, | |
385 | &bridge->mode); | |
386 | ||
387 | pci_set_drvdata(pdev, bridge); | |
388 | return agp_add_bridge(bridge); | |
389 | } | |
390 | ||
391 | static void __devexit agp_nvidia_remove(struct pci_dev *pdev) | |
392 | { | |
393 | struct agp_bridge_data *bridge = pci_get_drvdata(pdev); | |
394 | ||
395 | agp_remove_bridge(bridge); | |
396 | agp_put_bridge(bridge); | |
397 | } | |
398 | ||
c5f2f261 DJ |
399 | #ifdef CONFIG_PM |
400 | static int agp_nvidia_suspend(struct pci_dev *pdev, pm_message_t state) | |
401 | { | |
402 | pci_save_state (pdev); | |
403 | pci_set_power_state (pdev, 3); | |
404 | ||
405 | return 0; | |
406 | } | |
407 | ||
408 | static int agp_nvidia_resume(struct pci_dev *pdev) | |
409 | { | |
c5f2f261 DJ |
410 | /* set power state 0 and restore PCI space */ |
411 | pci_set_power_state (pdev, 0); | |
412 | pci_restore_state(pdev); | |
413 | ||
414 | /* reconfigure AGP hardware again */ | |
415 | nvidia_configure(); | |
416 | ||
417 | return 0; | |
418 | } | |
419 | #endif | |
420 | ||
421 | ||
1da177e4 LT |
422 | static struct pci_device_id agp_nvidia_pci_table[] = { |
423 | { | |
424 | .class = (PCI_CLASS_BRIDGE_HOST << 8), | |
425 | .class_mask = ~0, | |
426 | .vendor = PCI_VENDOR_ID_NVIDIA, | |
427 | .device = PCI_DEVICE_ID_NVIDIA_NFORCE, | |
428 | .subvendor = PCI_ANY_ID, | |
429 | .subdevice = PCI_ANY_ID, | |
430 | }, | |
431 | { | |
432 | .class = (PCI_CLASS_BRIDGE_HOST << 8), | |
433 | .class_mask = ~0, | |
434 | .vendor = PCI_VENDOR_ID_NVIDIA, | |
435 | .device = PCI_DEVICE_ID_NVIDIA_NFORCE2, | |
436 | .subvendor = PCI_ANY_ID, | |
437 | .subdevice = PCI_ANY_ID, | |
438 | }, | |
439 | { } | |
440 | }; | |
441 | ||
442 | MODULE_DEVICE_TABLE(pci, agp_nvidia_pci_table); | |
443 | ||
444 | static struct pci_driver agp_nvidia_pci_driver = { | |
445 | .name = "agpgart-nvidia", | |
446 | .id_table = agp_nvidia_pci_table, | |
447 | .probe = agp_nvidia_probe, | |
448 | .remove = agp_nvidia_remove, | |
c5f2f261 DJ |
449 | #ifdef CONFIG_PM |
450 | .suspend = agp_nvidia_suspend, | |
451 | .resume = agp_nvidia_resume, | |
452 | #endif | |
1da177e4 LT |
453 | }; |
454 | ||
455 | static int __init agp_nvidia_init(void) | |
456 | { | |
457 | if (agp_off) | |
458 | return -EINVAL; | |
459 | return pci_register_driver(&agp_nvidia_pci_driver); | |
460 | } | |
461 | ||
462 | static void __exit agp_nvidia_cleanup(void) | |
463 | { | |
464 | pci_unregister_driver(&agp_nvidia_pci_driver); | |
1e415732 AC |
465 | pci_dev_put(nvidia_private.dev_1); |
466 | pci_dev_put(nvidia_private.dev_2); | |
467 | pci_dev_put(nvidia_private.dev_3); | |
1da177e4 LT |
468 | } |
469 | ||
470 | module_init(agp_nvidia_init); | |
471 | module_exit(agp_nvidia_cleanup); | |
472 | ||
473 | MODULE_LICENSE("GPL and additional rights"); | |
474 | MODULE_AUTHOR("NVIDIA Corporation"); | |
475 |