[PATCH] freepgt: hugetlb_free_pgd_range

ia64 and ppc64 had hugetlb_free_pgtables functions which were no longer being
called, and it wasn't obvious what to do about them.

The ppc64 case turns out to be easy: the associated tables are noted elsewhere
and freed later, safe to either skip its hugetlb areas or go through the
motions of freeing nothing.  Since ia64 does need a special case, restore to
ppc64 the special case of skipping them.

The ia64 hugetlb case has been broken since pgd_addr_end went in, though it
probably appeared to work okay if you just had one such area; in fact it's
been broken much longer if you consider a long munmap spanning from another
region into the hugetlb region.

In the ia64 hugetlb region, more virtual address bits are available than in
the other regions, yet the page tables are structured the same way: the page
at the bottom is larger.  Here we need to scale down each addr before passing
it to the standard free_pgd_range.  Was about to write a hugely_scaled_down
macro, but found htlbpage_to_page already exists for just this purpose.  Fixed
off-by-one in ia64 is_hugepage_only_range.

Uninline free_pgd_range to make it available to ia64.  Make sure the
vma-gathering loop in free_pgtables cannot join a hugepage_only_range to any
other (safe to join huges?  probably but don't bother).

Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index 626258ae974290d354c5384b725ffac8647f55c2..df08ae7634b61d2efadf065e246c4bfb7f342b2c 100644 (file)
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -186,13 +186,30 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int wri
        return NULL;
 }
 
-/*
- * Do nothing, until we've worked out what to do!  To allow build, we
- * must remove reference to clear_page_range since it no longer exists.
- */
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-       unsigned long start, unsigned long end)
+void hugetlb_free_pgd_range(struct mmu_gather **tlb,
+                       unsigned long addr, unsigned long end,
+                       unsigned long floor, unsigned long ceiling)
 {
+       /*
+        * This is called only when is_hugepage_only_range(addr,),
+        * and it follows that is_hugepage_only_range(end,) also.
+        *
+        * The offset of these addresses from the base of the hugetlb
+        * region must be scaled down by HPAGE_SIZE/PAGE_SIZE so that
+        * the standard free_pgd_range will free the right page tables.
+        *
+        * If floor and ceiling are also in the hugetlb region, they
+        * must likewise be scaled down; but if outside, left unchanged.
+        */
+
+       addr = htlbpage_to_page(addr);
+       end  = htlbpage_to_page(end);
+       if (is_hugepage_only_range(tlb->mm, floor, HPAGE_SIZE))
+               floor = htlbpage_to_page(floor);
+       if (is_hugepage_only_range(tlb->mm, ceiling, HPAGE_SIZE))
+               ceiling = htlbpage_to_page(ceiling);
+
+       free_pgd_range(tlb, addr, end, floor, ceiling);
 }
 
 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)

diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c
index c62ddaff07209efb6d099d5d94f9306be5d70f72..8665bb57e42bb77349276a36ce82b98e13ea3fac 100644 (file)
--- a/arch/ppc64/mm/hugetlbpage.c
+++ b/arch/ppc64/mm/hugetlbpage.c
@@ -430,16 +430,6 @@ void unmap_hugepage_range(struct vm_area_struct *vma,
        flush_tlb_pending();
 }
 
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-                          unsigned long start, unsigned long end)
-{
-       /* Because the huge pgtables are only 2 level, they can take
-        * at most around 4M, much less than one hugepage which the
-        * process is presumably entitled to use.  So we don't bother
-        * freeing up the pagetables on unmap, and wait until
-        * destroy_context() to clean up the lot. */
-}
-
 int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
 {
        struct mm_struct *mm = current->mm;

diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h
index 24aab801a8caa60db0a48eb86c7cc5a2caeb03be..08894f73abf0384df70d33888228ee389be33bcb 100644 (file)
--- a/include/asm-ia64/page.h
+++ b/include/asm-ia64/page.h
@@ -139,7 +139,7 @@ typedef union ia64_va {
 # define HUGETLB_PAGE_ORDER    (HPAGE_SHIFT - PAGE_SHIFT)
 # define is_hugepage_only_range(mm, addr, len)         \
         (REGION_NUMBER(addr) == REGION_HPAGE &&        \
-         REGION_NUMBER((addr)+(len)) == REGION_HPAGE)
+         REGION_NUMBER((addr)+(len)-1) == REGION_HPAGE)
 extern unsigned int hpage_shift;
 #endif
 

diff --git a/include/asm-ia64/pgtable.h b/include/asm-ia64/pgtable.h
index 1757a811f4361590cca5475e8e89c5fea6d25d1a..bbf6dd7570038e2d26f135fd52ee9f33c0c9f69d 100644 (file)
--- a/include/asm-ia64/pgtable.h
+++ b/include/asm-ia64/pgtable.h
@@ -472,8 +472,8 @@ extern struct page *zero_page_memmap_ptr;
 #define HUGETLB_PGDIR_SIZE     (__IA64_UL(1) << HUGETLB_PGDIR_SHIFT)
 #define HUGETLB_PGDIR_MASK     (~(HUGETLB_PGDIR_SIZE-1))
 struct mmu_gather;
-extern void hugetlb_free_pgtables(struct mmu_gather *tlb,
-       struct vm_area_struct * prev, unsigned long start, unsigned long end);
+void hugetlb_free_pgd_range(struct mmu_gather **tlb, unsigned long addr,
+               unsigned long end, unsigned long floor, unsigned long ceiling);
 #endif
 
 /*

diff --git a/include/asm-ppc64/pgtable.h b/include/asm-ppc64/pgtable.h
index 4c4824653e80944f8d48d3b9b3ce201cfc8390e3..33b90e2aa47d88074ea8d32bbd2d4872642aa783 100644 (file)
--- a/include/asm-ppc64/pgtable.h
+++ b/include/asm-ppc64/pgtable.h
@@ -500,9 +500,15 @@ extern pgd_t ioremap_dir[1024];
 
 extern void paging_init(void);
 
-struct mmu_gather;
-void hugetlb_free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
-                          unsigned long start, unsigned long end);
+/*
+ * Because the huge pgtables are only 2 level, they can take
+ * at most around 4M, much less than one hugepage which the
+ * process is presumably entitled to use.  So we don't bother
+ * freeing up the pagetables on unmap, and wait until
+ * destroy_context() to clean up the lot.
+ */
+#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \
+                                               do { } while (0)
 
 /*
  * This gets called at the end of handling a page fault, when

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index ae45676d27baf3cae11310a88e56cca610747875..6af1ae4a821196a54981887cecc2fe32c0e18241 100644 (file)
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -37,7 +37,8 @@ extern int sysctl_hugetlb_shm_group;
 
 #ifndef ARCH_HAS_HUGEPAGE_ONLY_RANGE
 #define is_hugepage_only_range(mm, addr, len)  0
-#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0)
+#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \
+                                               do { } while (0)
 #endif
 
 #ifndef ARCH_HAS_PREPARE_HUGEPAGE_RANGE
@@ -72,7 +73,8 @@ static inline unsigned long hugetlb_total_pages(void)
 #define prepare_hugepage_range(addr, len)      (-EINVAL)
 #define pmd_huge(x)    0
 #define is_hugepage_only_range(mm, addr, len)  0
-#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0)
+#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) \
+                                               do { } while (0)
 #define alloc_huge_page()                      ({ NULL; })
 #define free_huge_page(p)                      ({ (void)(p); BUG(); })
 

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 59eca28b5ae2d64d9d21a2b0cff152b47ca36152..c74a74ca401dbeae3752e9f201f331505f4f9b48 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -587,7 +587,9 @@ unsigned long unmap_vmas(struct mmu_gather **tlb, struct mm_struct *mm,
                struct vm_area_struct *start_vma, unsigned long start_addr,
                unsigned long end_addr, unsigned long *nr_accounted,
                struct zap_details *);
-void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
+void free_pgd_range(struct mmu_gather **tlb, unsigned long addr,
+               unsigned long end, unsigned long floor, unsigned long ceiling);
+void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *start_vma,
                unsigned long floor, unsigned long ceiling);
 int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
                        struct vm_area_struct *vma);

diff --git a/mm/memory.c b/mm/memory.c
index 854bd90eeca1928b15dfb61088dd3d13360cb373..6bad4c4064e7e0bae9886d976ccd7fbde8ccf753 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -190,7 +190,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
  *
  * Must be called with pagetable lock held.
  */
-static inline void free_pgd_range(struct mmu_gather *tlb,
+void free_pgd_range(struct mmu_gather **tlb,
                        unsigned long addr, unsigned long end,
                        unsigned long floor, unsigned long ceiling)
 {
@@ -241,37 +241,47 @@ static inline void free_pgd_range(struct mmu_gather *tlb,
                return;
 
        start = addr;
-       pgd = pgd_offset(tlb->mm, addr);
+       pgd = pgd_offset((*tlb)->mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd))
                        continue;
-               free_pud_range(tlb, pgd, addr, next, floor, ceiling);
+               free_pud_range(*tlb, pgd, addr, next, floor, ceiling);
        } while (pgd++, addr = next, addr != end);
 
-       if (!tlb_is_full_mm(tlb))
-               flush_tlb_pgtables(tlb->mm, start, end);
+       if (!tlb_is_full_mm(*tlb))
+               flush_tlb_pgtables((*tlb)->mm, start, end);
 }
 
 void free_pgtables(struct mmu_gather **tlb, struct vm_area_struct *vma,
-                               unsigned long floor, unsigned long ceiling)
+               unsigned long floor, unsigned long ceiling)
 {
        while (vma) {
                struct vm_area_struct *next = vma->vm_next;
                unsigned long addr = vma->vm_start;
 
-               /* Optimization: gather nearby vmas into a single call down */
-               while (next && next->vm_start <= vma->vm_end + PMD_SIZE) {
-                       vma = next;
-                       next = vma->vm_next;
-               }
-               free_pgd_range(*tlb, addr, vma->vm_end,
+               if (is_hugepage_only_range(vma->vm_mm, addr, HPAGE_SIZE)) {
+                       hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
+                               floor, next? next->vm_start: ceiling);
+               } else {
+                       /*
+                        * Optimization: gather nearby vmas into one call down
+                        */
+                       while (next && next->vm_start <= vma->vm_end + PMD_SIZE
+                         && !is_hugepage_only_range(vma->vm_mm, next->vm_start,
+                                                       HPAGE_SIZE)) {
+                               vma = next;
+                               next = vma->vm_next;
+                       }
+                       free_pgd_range(tlb, addr, vma->vm_end,
                                floor, next? next->vm_start: ceiling);
+               }
                vma = next;
        }
 }
 
-pte_t fastcall * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
+pte_t fastcall *pte_alloc_map(struct mm_struct *mm, pmd_t *pmd,
+                               unsigned long address)
 {
        if (!pmd_present(*pmd)) {
                struct page *new;