Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Nvidia AGPGART routines. | |
3 | * Based upon a 2.4 agpgart diff by the folks from NVIDIA, and hacked up | |
f4432c5c | 4 | * to work in 2.5 by Dave Jones <davej@redhat.com> |
1da177e4 LT |
5 | */ |
6 | ||
7 | #include <linux/module.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/init.h> | |
10 | #include <linux/agp_backend.h> | |
1da177e4 LT |
11 | #include <linux/page-flags.h> |
12 | #include <linux/mm.h> | |
05613bdd | 13 | #include <linux/jiffies.h> |
1da177e4 LT |
14 | #include "agp.h" |
15 | ||
16 | /* NVIDIA registers */ | |
17 | #define NVIDIA_0_APSIZE 0x80 | |
18 | #define NVIDIA_1_WBC 0xf0 | |
19 | #define NVIDIA_2_GARTCTRL 0xd0 | |
20 | #define NVIDIA_2_APBASE 0xd8 | |
21 | #define NVIDIA_2_APLIMIT 0xdc | |
22 | #define NVIDIA_2_ATTBASE(i) (0xe0 + (i) * 4) | |
23 | #define NVIDIA_3_APBASE 0x50 | |
24 | #define NVIDIA_3_APLIMIT 0x54 | |
25 | ||
26 | ||
27 | static struct _nvidia_private { | |
28 | struct pci_dev *dev_1; | |
29 | struct pci_dev *dev_2; | |
30 | struct pci_dev *dev_3; | |
31 | volatile u32 __iomem *aperture; | |
32 | int num_active_entries; | |
33 | off_t pg_offset; | |
34 | u32 wbc_mask; | |
35 | } nvidia_private; | |
36 | ||
37 | ||
38 | static int nvidia_fetch_size(void) | |
39 | { | |
40 | int i; | |
41 | u8 size_value; | |
42 | struct aper_size_info_8 *values; | |
43 | ||
44 | pci_read_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, &size_value); | |
45 | size_value &= 0x0f; | |
46 | values = A_SIZE_8(agp_bridge->driver->aperture_sizes); | |
47 | ||
48 | for (i = 0; i < agp_bridge->driver->num_aperture_sizes; i++) { | |
49 | if (size_value == values[i].size_value) { | |
50 | agp_bridge->previous_size = | |
51 | agp_bridge->current_size = (void *) (values + i); | |
52 | agp_bridge->aperture_size_idx = i; | |
53 | return values[i].size; | |
54 | } | |
55 | } | |
56 | ||
57 | return 0; | |
58 | } | |
59 | ||
60 | #define SYSCFG 0xC0010010 | |
61 | #define IORR_BASE0 0xC0010016 | |
62 | #define IORR_MASK0 0xC0010017 | |
63 | #define AMD_K7_NUM_IORR 2 | |
64 | ||
65 | static int nvidia_init_iorr(u32 base, u32 size) | |
66 | { | |
67 | u32 base_hi, base_lo; | |
68 | u32 mask_hi, mask_lo; | |
69 | u32 sys_hi, sys_lo; | |
70 | u32 iorr_addr, free_iorr_addr; | |
71 | ||
72 | /* Find the iorr that is already used for the base */ | |
73 | /* If not found, determine the uppermost available iorr */ | |
74 | free_iorr_addr = AMD_K7_NUM_IORR; | |
6a92a4e0 | 75 | for (iorr_addr = 0; iorr_addr < AMD_K7_NUM_IORR; iorr_addr++) { |
1da177e4 LT |
76 | rdmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi); |
77 | rdmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi); | |
78 | ||
79 | if ((base_lo & 0xfffff000) == (base & 0xfffff000)) | |
80 | break; | |
81 | ||
82 | if ((mask_lo & 0x00000800) == 0) | |
83 | free_iorr_addr = iorr_addr; | |
84 | } | |
6a92a4e0 | 85 | |
1da177e4 LT |
86 | if (iorr_addr >= AMD_K7_NUM_IORR) { |
87 | iorr_addr = free_iorr_addr; | |
88 | if (iorr_addr >= AMD_K7_NUM_IORR) | |
89 | return -EINVAL; | |
90 | } | |
91 | base_hi = 0x0; | |
92 | base_lo = (base & ~0xfff) | 0x18; | |
93 | mask_hi = 0xf; | |
94 | mask_lo = ((~(size - 1)) & 0xfffff000) | 0x800; | |
95 | wrmsr(IORR_BASE0 + 2 * iorr_addr, base_lo, base_hi); | |
96 | wrmsr(IORR_MASK0 + 2 * iorr_addr, mask_lo, mask_hi); | |
97 | ||
98 | rdmsr(SYSCFG, sys_lo, sys_hi); | |
99 | sys_lo |= 0x00100000; | |
100 | wrmsr(SYSCFG, sys_lo, sys_hi); | |
101 | ||
102 | return 0; | |
103 | } | |
104 | ||
105 | static int nvidia_configure(void) | |
106 | { | |
107 | int i, rc, num_dirs; | |
108 | u32 apbase, aplimit; | |
109 | struct aper_size_info_8 *current_size; | |
110 | u32 temp; | |
111 | ||
112 | current_size = A_SIZE_8(agp_bridge->current_size); | |
113 | ||
114 | /* aperture size */ | |
115 | pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, | |
116 | current_size->size_value); | |
117 | ||
118 | /* address to map to */ | |
119 | pci_read_config_dword(agp_bridge->dev, AGP_APBASE, &apbase); | |
120 | apbase &= PCI_BASE_ADDRESS_MEM_MASK; | |
121 | agp_bridge->gart_bus_addr = apbase; | |
122 | aplimit = apbase + (current_size->size * 1024 * 1024) - 1; | |
123 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APBASE, apbase); | |
124 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_APLIMIT, aplimit); | |
125 | pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APBASE, apbase); | |
126 | pci_write_config_dword(nvidia_private.dev_3, NVIDIA_3_APLIMIT, aplimit); | |
127 | if (0 != (rc = nvidia_init_iorr(apbase, current_size->size * 1024 * 1024))) | |
128 | return rc; | |
129 | ||
130 | /* directory size is 64k */ | |
131 | num_dirs = current_size->size / 64; | |
132 | nvidia_private.num_active_entries = current_size->num_entries; | |
133 | nvidia_private.pg_offset = 0; | |
134 | if (num_dirs == 0) { | |
135 | num_dirs = 1; | |
136 | nvidia_private.num_active_entries /= (64 / current_size->size); | |
137 | nvidia_private.pg_offset = (apbase & (64 * 1024 * 1024 - 1) & | |
138 | ~(current_size->size * 1024 * 1024 - 1)) / PAGE_SIZE; | |
139 | } | |
140 | ||
141 | /* attbase */ | |
6a92a4e0 | 142 | for (i = 0; i < 8; i++) { |
1da177e4 LT |
143 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_ATTBASE(i), |
144 | (agp_bridge->gatt_bus_addr + (i % num_dirs) * 64 * 1024) | 1); | |
145 | } | |
146 | ||
147 | /* gtlb control */ | |
148 | pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp); | |
149 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp | 0x11); | |
150 | ||
151 | /* gart control */ | |
152 | pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp); | |
153 | pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp | 0x100); | |
154 | ||
155 | /* map aperture */ | |
156 | nvidia_private.aperture = | |
157 | (volatile u32 __iomem *) ioremap(apbase, 33 * PAGE_SIZE); | |
158 | ||
5bdbc7dc ST |
159 | if (!nvidia_private.aperture) |
160 | return -ENOMEM; | |
161 | ||
1da177e4 LT |
162 | return 0; |
163 | } | |
164 | ||
165 | static void nvidia_cleanup(void) | |
166 | { | |
167 | struct aper_size_info_8 *previous_size; | |
168 | u32 temp; | |
169 | ||
170 | /* gart control */ | |
171 | pci_read_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, &temp); | |
172 | pci_write_config_dword(agp_bridge->dev, NVIDIA_0_APSIZE, temp & ~(0x100)); | |
173 | ||
174 | /* gtlb control */ | |
175 | pci_read_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, &temp); | |
176 | pci_write_config_dword(nvidia_private.dev_2, NVIDIA_2_GARTCTRL, temp & ~(0x11)); | |
177 | ||
178 | /* unmap aperture */ | |
179 | iounmap((void __iomem *) nvidia_private.aperture); | |
180 | ||
181 | /* restore previous aperture size */ | |
182 | previous_size = A_SIZE_8(agp_bridge->previous_size); | |
183 | pci_write_config_byte(agp_bridge->dev, NVIDIA_0_APSIZE, | |
184 | previous_size->size_value); | |
185 | ||
186 | /* restore iorr for previous aperture size */ | |
187 | nvidia_init_iorr(agp_bridge->gart_bus_addr, | |
188 | previous_size->size * 1024 * 1024); | |
189 | } | |
190 | ||
191 | ||
192 | /* | |
193 | * Note we can't use the generic routines, even though they are 99% the same. | |
194 | * Aperture sizes <64M still requires a full 64k GART directory, but | |
195 | * only use the portion of the TLB entries that correspond to the apertures | |
196 | * alignment inside the surrounding 64M block. | |
197 | */ | |
198 | extern int agp_memory_reserved; | |
199 | ||
200 | static int nvidia_insert_memory(struct agp_memory *mem, off_t pg_start, int type) | |
201 | { | |
202 | int i, j; | |
a64d2b37 | 203 | int mask_type; |
6a92a4e0 | 204 | |
a64d2b37 TH |
205 | mask_type = agp_generic_type_to_mask_type(mem->bridge, type); |
206 | if (mask_type != 0 || type != mem->type) | |
1da177e4 | 207 | return -EINVAL; |
6a92a4e0 | 208 | |
a64d2b37 TH |
209 | if (mem->page_count == 0) |
210 | return 0; | |
211 | ||
1da177e4 LT |
212 | if ((pg_start + mem->page_count) > |
213 | (nvidia_private.num_active_entries - agp_memory_reserved/PAGE_SIZE)) | |
214 | return -EINVAL; | |
6a92a4e0 DJ |
215 | |
216 | for (j = pg_start; j < (pg_start + mem->page_count); j++) { | |
1da177e4 LT |
217 | if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j))) |
218 | return -EBUSY; | |
219 | } | |
220 | ||
c7258012 | 221 | if (!mem->is_flushed) { |
1da177e4 | 222 | global_cache_flush(); |
c7258012 | 223 | mem->is_flushed = true; |
1da177e4 LT |
224 | } |
225 | for (i = 0, j = pg_start; i < mem->page_count; i++, j++) { | |
226 | writel(agp_bridge->driver->mask_memory(agp_bridge, | |
6a12235c | 227 | page_to_phys(mem->pages[i]), mask_type), |
1da177e4 | 228 | agp_bridge->gatt_table+nvidia_private.pg_offset+j); |
1da177e4 | 229 | } |
a64d2b37 TH |
230 | |
231 | /* PCI Posting. */ | |
232 | readl(agp_bridge->gatt_table+nvidia_private.pg_offset+j - 1); | |
233 | ||
1da177e4 LT |
234 | agp_bridge->driver->tlb_flush(mem); |
235 | return 0; | |
236 | } | |
237 | ||
238 | ||
239 | static int nvidia_remove_memory(struct agp_memory *mem, off_t pg_start, int type) | |
240 | { | |
241 | int i; | |
242 | ||
a64d2b37 TH |
243 | int mask_type; |
244 | ||
245 | mask_type = agp_generic_type_to_mask_type(mem->bridge, type); | |
246 | if (mask_type != 0 || type != mem->type) | |
1da177e4 LT |
247 | return -EINVAL; |
248 | ||
a64d2b37 TH |
249 | if (mem->page_count == 0) |
250 | return 0; | |
251 | ||
1da177e4 LT |
252 | for (i = pg_start; i < (mem->page_count + pg_start); i++) |
253 | writel(agp_bridge->scratch_page, agp_bridge->gatt_table+nvidia_private.pg_offset+i); | |
254 | ||
255 | agp_bridge->driver->tlb_flush(mem); | |
256 | return 0; | |
257 | } | |
258 | ||
259 | ||
260 | static void nvidia_tlbflush(struct agp_memory *mem) | |
261 | { | |
262 | unsigned long end; | |
263 | u32 wbc_reg, temp; | |
264 | int i; | |
265 | ||
266 | /* flush chipset */ | |
267 | if (nvidia_private.wbc_mask) { | |
268 | pci_read_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, &wbc_reg); | |
269 | wbc_reg |= nvidia_private.wbc_mask; | |
270 | pci_write_config_dword(nvidia_private.dev_1, NVIDIA_1_WBC, wbc_reg); | |
271 | ||
272 | end = jiffies + 3*HZ; | |
273 | do { | |
274 | pci_read_config_dword(nvidia_private.dev_1, | |
275 | NVIDIA_1_WBC, &wbc_reg); | |
05613bdd | 276 | if (time_before_eq(end, jiffies)) { |
1da177e4 LT |
277 | printk(KERN_ERR PFX |
278 | "TLB flush took more than 3 seconds.\n"); | |
279 | } | |
280 | } while (wbc_reg & nvidia_private.wbc_mask); | |
281 | } | |
282 | ||
283 | /* flush TLB entries */ | |
6a92a4e0 | 284 | for (i = 0; i < 32 + 1; i++) |
1da177e4 | 285 | temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32))); |
6a92a4e0 | 286 | for (i = 0; i < 32 + 1; i++) |
1da177e4 LT |
287 | temp = readl(nvidia_private.aperture+(i * PAGE_SIZE / sizeof(u32))); |
288 | } | |
289 | ||
290 | ||
e5524f35 | 291 | static const struct aper_size_info_8 nvidia_generic_sizes[5] = |
1da177e4 LT |
292 | { |
293 | {512, 131072, 7, 0}, | |
294 | {256, 65536, 6, 8}, | |
295 | {128, 32768, 5, 12}, | |
296 | {64, 16384, 4, 14}, | |
297 | /* The 32M mode still requires a 64k gatt */ | |
298 | {32, 16384, 4, 15} | |
299 | }; | |
300 | ||
301 | ||
e5524f35 | 302 | static const struct gatt_mask nvidia_generic_masks[] = |
1da177e4 LT |
303 | { |
304 | { .mask = 1, .type = 0} | |
305 | }; | |
306 | ||
307 | ||
e5524f35 | 308 | static const struct agp_bridge_driver nvidia_driver = { |
1da177e4 LT |
309 | .owner = THIS_MODULE, |
310 | .aperture_sizes = nvidia_generic_sizes, | |
311 | .size_type = U8_APER_SIZE, | |
312 | .num_aperture_sizes = 5, | |
61cf0593 | 313 | .needs_scratch_page = true, |
1da177e4 LT |
314 | .configure = nvidia_configure, |
315 | .fetch_size = nvidia_fetch_size, | |
316 | .cleanup = nvidia_cleanup, | |
317 | .tlb_flush = nvidia_tlbflush, | |
318 | .mask_memory = agp_generic_mask_memory, | |
319 | .masks = nvidia_generic_masks, | |
320 | .agp_enable = agp_generic_enable, | |
321 | .cache_flush = global_cache_flush, | |
322 | .create_gatt_table = agp_generic_create_gatt_table, | |
323 | .free_gatt_table = agp_generic_free_gatt_table, | |
324 | .insert_memory = nvidia_insert_memory, | |
325 | .remove_memory = nvidia_remove_memory, | |
326 | .alloc_by_type = agp_generic_alloc_by_type, | |
327 | .free_by_type = agp_generic_free_by_type, | |
328 | .agp_alloc_page = agp_generic_alloc_page, | |
5f310b63 | 329 | .agp_alloc_pages = agp_generic_alloc_pages, |
1da177e4 | 330 | .agp_destroy_page = agp_generic_destroy_page, |
5f310b63 | 331 | .agp_destroy_pages = agp_generic_destroy_pages, |
a030ce44 | 332 | .agp_type_to_mask_type = agp_generic_type_to_mask_type, |
1da177e4 LT |
333 | }; |
334 | ||
bcd2982a GKH |
335 | static int agp_nvidia_probe(struct pci_dev *pdev, |
336 | const struct pci_device_id *ent) | |
1da177e4 LT |
337 | { |
338 | struct agp_bridge_data *bridge; | |
339 | u8 cap_ptr; | |
340 | ||
341 | nvidia_private.dev_1 = | |
1e415732 | 342 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 1)); |
1da177e4 | 343 | nvidia_private.dev_2 = |
1e415732 | 344 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 2)); |
1da177e4 | 345 | nvidia_private.dev_3 = |
1e415732 | 346 | pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(30, 0)); |
6a92a4e0 | 347 | |
1da177e4 LT |
348 | if (!nvidia_private.dev_1 || !nvidia_private.dev_2 || !nvidia_private.dev_3) { |
349 | printk(KERN_INFO PFX "Detected an NVIDIA nForce/nForce2 " | |
350 | "chipset, but could not find the secondary devices.\n"); | |
351 | return -ENODEV; | |
352 | } | |
353 | ||
354 | cap_ptr = pci_find_capability(pdev, PCI_CAP_ID_AGP); | |
355 | if (!cap_ptr) | |
356 | return -ENODEV; | |
357 | ||
358 | switch (pdev->device) { | |
359 | case PCI_DEVICE_ID_NVIDIA_NFORCE: | |
360 | printk(KERN_INFO PFX "Detected NVIDIA nForce chipset\n"); | |
361 | nvidia_private.wbc_mask = 0x00010000; | |
362 | break; | |
363 | case PCI_DEVICE_ID_NVIDIA_NFORCE2: | |
364 | printk(KERN_INFO PFX "Detected NVIDIA nForce2 chipset\n"); | |
365 | nvidia_private.wbc_mask = 0x80000000; | |
366 | break; | |
367 | default: | |
368 | printk(KERN_ERR PFX "Unsupported NVIDIA chipset (device id: %04x)\n", | |
369 | pdev->device); | |
370 | return -ENODEV; | |
371 | } | |
372 | ||
373 | bridge = agp_alloc_bridge(); | |
374 | if (!bridge) | |
375 | return -ENOMEM; | |
376 | ||
377 | bridge->driver = &nvidia_driver; | |
378 | bridge->dev_private_data = &nvidia_private, | |
379 | bridge->dev = pdev; | |
380 | bridge->capndx = cap_ptr; | |
381 | ||
382 | /* Fill in the mode register */ | |
383 | pci_read_config_dword(pdev, | |
384 | bridge->capndx+PCI_AGP_STATUS, | |
385 | &bridge->mode); | |
386 | ||
387 | pci_set_drvdata(pdev, bridge); | |
388 | return agp_add_bridge(bridge); | |
389 | } | |
390 | ||
39af33fc | 391 | static void agp_nvidia_remove(struct pci_dev *pdev) |
1da177e4 LT |
392 | { |
393 | struct agp_bridge_data *bridge = pci_get_drvdata(pdev); | |
394 | ||
395 | agp_remove_bridge(bridge); | |
396 | agp_put_bridge(bridge); | |
397 | } | |
398 | ||
c5f2f261 DJ |
399 | #ifdef CONFIG_PM |
400 | static int agp_nvidia_suspend(struct pci_dev *pdev, pm_message_t state) | |
401 | { | |
402 | pci_save_state (pdev); | |
403 | pci_set_power_state (pdev, 3); | |
404 | ||
405 | return 0; | |
406 | } | |
407 | ||
408 | static int agp_nvidia_resume(struct pci_dev *pdev) | |
409 | { | |
c5f2f261 DJ |
410 | /* set power state 0 and restore PCI space */ |
411 | pci_set_power_state (pdev, 0); | |
412 | pci_restore_state(pdev); | |
413 | ||
414 | /* reconfigure AGP hardware again */ | |
415 | nvidia_configure(); | |
416 | ||
417 | return 0; | |
418 | } | |
419 | #endif | |
420 | ||
421 | ||
1da177e4 LT |
422 | static struct pci_device_id agp_nvidia_pci_table[] = { |
423 | { | |
424 | .class = (PCI_CLASS_BRIDGE_HOST << 8), | |
425 | .class_mask = ~0, | |
426 | .vendor = PCI_VENDOR_ID_NVIDIA, | |
427 | .device = PCI_DEVICE_ID_NVIDIA_NFORCE, | |
428 | .subvendor = PCI_ANY_ID, | |
429 | .subdevice = PCI_ANY_ID, | |
430 | }, | |
431 | { | |
432 | .class = (PCI_CLASS_BRIDGE_HOST << 8), | |
433 | .class_mask = ~0, | |
434 | .vendor = PCI_VENDOR_ID_NVIDIA, | |
435 | .device = PCI_DEVICE_ID_NVIDIA_NFORCE2, | |
436 | .subvendor = PCI_ANY_ID, | |
437 | .subdevice = PCI_ANY_ID, | |
438 | }, | |
439 | { } | |
440 | }; | |
441 | ||
442 | MODULE_DEVICE_TABLE(pci, agp_nvidia_pci_table); | |
443 | ||
444 | static struct pci_driver agp_nvidia_pci_driver = { | |
445 | .name = "agpgart-nvidia", | |
446 | .id_table = agp_nvidia_pci_table, | |
447 | .probe = agp_nvidia_probe, | |
448 | .remove = agp_nvidia_remove, | |
c5f2f261 DJ |
449 | #ifdef CONFIG_PM |
450 | .suspend = agp_nvidia_suspend, | |
451 | .resume = agp_nvidia_resume, | |
452 | #endif | |
1da177e4 LT |
453 | }; |
454 | ||
455 | static int __init agp_nvidia_init(void) | |
456 | { | |
457 | if (agp_off) | |
458 | return -EINVAL; | |
459 | return pci_register_driver(&agp_nvidia_pci_driver); | |
460 | } | |
461 | ||
462 | static void __exit agp_nvidia_cleanup(void) | |
463 | { | |
464 | pci_unregister_driver(&agp_nvidia_pci_driver); | |
1e415732 AC |
465 | pci_dev_put(nvidia_private.dev_1); |
466 | pci_dev_put(nvidia_private.dev_2); | |
467 | pci_dev_put(nvidia_private.dev_3); | |
1da177e4 LT |
468 | } |
469 | ||
470 | module_init(agp_nvidia_init); | |
471 | module_exit(agp_nvidia_cleanup); | |
472 | ||
473 | MODULE_LICENSE("GPL and additional rights"); | |
474 | MODULE_AUTHOR("NVIDIA Corporation"); | |
475 |