sparc64: Support huge PUD case in get_user_pages

get_user_pages() is used to do direct IO. It already
handles the case where the address range is backed
by PMD huge pages. This patch now adds the case where
the range could be backed by PUD huge pages.

Signed-off-by: Nitin Gupta <nitin.m.gupta@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 6fbd931f0570021fd297db9163ed170fc21aafe2..2579f5a22f462bfb216c1d9a3278eba855b736fb 100644 (file)
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -687,6 +687,8 @@ static inline unsigned long pmd_write(pmd_t pmd)
        return pte_write(pte);
 }
 
+#define pud_write(pud) pte_write(__pte(pud_val(pud)))
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 static inline unsigned long pmd_dirty(pmd_t pmd)
 {
@@ -823,9 +825,18 @@ static inline unsigned long __pmd_page(pmd_t pmd)
 
        return ((unsigned long) __va(pfn << PAGE_SHIFT));
 }
+
+static inline unsigned long pud_page_vaddr(pud_t pud)
+{
+       pte_t pte = __pte(pud_val(pud));
+       unsigned long pfn;
+
+       pfn = pte_pfn(pte);
+
+       return ((unsigned long) __va(pfn << PAGE_SHIFT));
+}
+
 #define pmd_page(pmd)                  virt_to_page((void *)__pmd_page(pmd))
-#define pud_page_vaddr(pud)            \
-       ((unsigned long) __va(pud_val(pud)))
 #define pud_page(pud)                  virt_to_page((void *)pud_page_vaddr(pud))
 #define pmd_clear(pmdp)                        (pmd_val(*(pmdp)) = 0UL)
 #define pud_present(pud)               (pud_val(pud) != 0U)

diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index f80cfc64c55ba2b975cf5c3c0fe4fb72da316de9..d809099ffd47394795b108d3ec7bed5efbcdc992 100644 (file)
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -103,6 +103,45 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
        return 1;
 }
 
+static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,
+                       unsigned long end, int write, struct page **pages,
+                       int *nr)
+{
+       struct page *head, *page;
+       int refs;
+
+       if (!(pud_val(pud) & _PAGE_VALID))
+               return 0;
+
+       if (write && !pud_write(pud))
+               return 0;
+
+       refs = 0;
+       page = pud_page(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
+       head = compound_head(page);
+       do {
+               VM_BUG_ON(compound_head(page) != head);
+               pages[*nr] = page;
+               (*nr)++;
+               page++;
+               refs++;
+       } while (addr += PAGE_SIZE, addr != end);
+
+       if (!page_cache_add_speculative(head, refs)) {
+               *nr -= refs;
+               return 0;
+       }
+
+       if (unlikely(pud_val(pud) != pud_val(*pudp))) {
+               *nr -= refs;
+               while (refs--)
+                       put_page(head);
+               return 0;
+       }
+
+       return 1;
+}
+
 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                int write, struct page **pages, int *nr)
 {
@@ -141,7 +180,11 @@ static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
                next = pud_addr_end(addr, end);
                if (pud_none(pud))
                        return 0;
-               if (!gup_pmd_range(pud, addr, next, write, pages, nr))
+               if (unlikely(pud_large(pud))) {
+                       if (!gup_huge_pud(pudp, pud, addr, next,
+                                         write, pages, nr))
+                               return 0;
+               } else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
                        return 0;
        } while (pudp++, addr = next, addr != end);