[PATCH] AGP performance fixes

AGP allocation/deallocation is suffering major performance issues due to
the nature of global_flush_tlb() being called on every change_page_attr()
call.

For small allocations this isn't really seen, but when you start allocating
50000 pages of AGP space, for say, texture memory, then things can take
seconds to complete.

In some cases the situation is doubled or even quadrupled in the time due
to SMP, or a deallocation, then a new reallocation.  I've had a case of
upto 20 seconds wait time to deallocate and reallocate AGP space.

This patch fixes the problem by making it the caller's responsibility to
call global_flush_tlb(), and so removes it from every instance of mapping a
page into AGP space until the time that all change_page_attr() changes are
done.

Signed-off-by: Dave Jones <davej@redhat.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>

diff --git a/drivers/char/agp/backend.c b/drivers/char/agp/backend.c
index 73f333f491bde21a3cd14656e15c8d7d740450b5..4d5ed18dad00455e938a93ef55254fe3abdb3758 100644 (file)
--- a/drivers/char/agp/backend.c
+++ b/drivers/char/agp/backend.c
@@ -147,6 +147,7 @@ static int agp_backend_initialize(struct agp_bridge_data *bridge)
                        printk(KERN_ERR PFX "unable to get memory for scratch page.\n");
                        return -ENOMEM;
                }
+               global_flush_tlb();
 
                bridge->scratch_page_real = virt_to_gart(addr);
                bridge->scratch_page =
@@ -187,9 +188,11 @@ static int agp_backend_initialize(struct agp_bridge_data *bridge)
        return 0;
 
 err_out:
-       if (bridge->driver->needs_scratch_page)
+       if (bridge->driver->needs_scratch_page) {
                bridge->driver->agp_destroy_page(
                                gart_to_virt(bridge->scratch_page_real));
+               global_flush_tlb();
+       }
        if (got_gatt)
                bridge->driver->free_gatt_table(bridge);
        if (got_keylist) {
@@ -211,9 +214,11 @@ static void agp_backend_cleanup(struct agp_bridge_data *bridge)
        bridge->key_list = NULL;
 
        if (bridge->driver->agp_destroy_page &&
-           bridge->driver->needs_scratch_page)
+           bridge->driver->needs_scratch_page) {
                bridge->driver->agp_destroy_page(
                                gart_to_virt(bridge->scratch_page_real));
+               global_flush_tlb();
+       }
 }
 
 /* When we remove the global variable agp_bridge from all drivers

diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index c4a38715c6f9751c3127497e2592b5717f1a2f32..19f242b5078169578a9f4d00bc30fe9d0e551748 100644 (file)
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -57,7 +57,8 @@ int map_page_into_agp(struct page *page)
 {
        int i;
        i = change_page_attr(page, 1, PAGE_KERNEL_NOCACHE);
-       global_flush_tlb();
+       /* Caller's responsibility to call global_flush_tlb() for
+        * performance reasons */
        return i;
 }
 EXPORT_SYMBOL_GPL(map_page_into_agp);
@@ -66,7 +67,8 @@ int unmap_page_from_agp(struct page *page)
 {
        int i;
        i = change_page_attr(page, 1, PAGE_KERNEL);
-       global_flush_tlb();
+       /* Caller's responsibility to call global_flush_tlb() for
+        * performance reasons */
        return i;
 }
 EXPORT_SYMBOL_GPL(unmap_page_from_agp);
@@ -153,6 +155,7 @@ void agp_free_memory(struct agp_memory *curr)
                for (i = 0; i < curr->page_count; i++) {
                        curr->bridge->driver->agp_destroy_page(gart_to_virt(curr->memory[i]));
                }
+               global_flush_tlb();
        }
        agp_free_key(curr->key);
        vfree(curr->memory);
@@ -210,7 +213,9 @@ struct agp_memory *agp_allocate_memory(struct agp_bridge_data *bridge,
                new->memory[i] = virt_to_gart(addr);
                new->page_count++;
        }
-       new->bridge = bridge;
+       global_flush_tlb();
+
+       new->bridge = bridge;
 
        flush_agp_mappings();
 

diff --git a/drivers/char/agp/i460-agp.c b/drivers/char/agp/i460-agp.c
index 58944cd271ead9c55120a43ffa014bd5f34ac9cf..be2e569ae9105ad6c5bbe6fc28770f3e98cbfd7b 100644 (file)
--- a/drivers/char/agp/i460-agp.c
+++ b/drivers/char/agp/i460-agp.c
@@ -514,9 +514,10 @@ static void *i460_alloc_page (struct agp_bridge_data *bridge)
 {
        void *page;
 
-       if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT)
+       if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT) {
                page = agp_generic_alloc_page(agp_bridge);
-       else
+               global_flush_tlb();
+       } else
                /* Returning NULL would cause problems */
                /* AK: really dubious code. */
                page = (void *)~0UL;
@@ -525,8 +526,10 @@ static void *i460_alloc_page (struct agp_bridge_data *bridge)
 
 static void i460_destroy_page (void *page)
 {
-       if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT)
+       if (I460_IO_PAGE_SHIFT <= PAGE_SHIFT) {
                agp_generic_destroy_page(page);
+               global_flush_tlb();
+       }
 }
 
 #endif /* I460_LARGE_IO_PAGES */

diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index bf4cc9ffd5b1dd737f0921b8319f3efeabdc2b77..027161ab88e9bff25321c8c1d5811ac0f8559e6b 100644 (file)
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -270,6 +270,7 @@ static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 
        switch (pg_count) {
        case 1: addr = agp_bridge->driver->agp_alloc_page(agp_bridge);
+               global_flush_tlb();
                break;
        case 4:
                /* kludge to get 4 physical pages for ARGB cursor */
@@ -330,9 +331,11 @@ static void intel_i810_free_by_type(struct agp_memory *curr)
        if(curr->type == AGP_PHYS_MEMORY) {
                if (curr->page_count == 4)
                        i8xx_destroy_pages(gart_to_virt(curr->memory[0]));
-               else
+               else {
                        agp_bridge->driver->agp_destroy_page(
                                 gart_to_virt(curr->memory[0]));
+                       global_flush_tlb();
+               }
                vfree(curr->memory);
        }
        kfree(curr);