thp: change split_huge_page_pmd() interface

Pass vma instead of mm and add address parameter.

In most cases we already have vma on the stack. We provides
split_huge_page_pmd_mm() for few cases when we have mm, but not vma.

This change is preparation to huge zero pmd splitting implementation.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: "H. Peter Anvin" <hpa@linux.intel.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/Documentation/vm/transhuge.txt b/Documentation/vm/transhuge.txt
index f734bb2a78dc797aa62a2d2b0e1e7cc3390fef87..8f5b41db314cfe0f9c6e1e978cdfd315ddf26019 100644 (file)
--- a/Documentation/vm/transhuge.txt
+++ b/Documentation/vm/transhuge.txt
@@ -276,7 +276,7 @@ unaffected. libhugetlbfs will also work fine as usual.
 == Graceful fallback ==
 
 Code walking pagetables but unware about huge pmds can simply call
-split_huge_page_pmd(mm, pmd) where the pmd is the one returned by
+split_huge_page_pmd(vma, addr, pmd) where the pmd is the one returned by
 pmd_offset. It's trivial to make the code transparent hugepage aware
 by just grepping for "pmd_offset" and adding split_huge_page_pmd where
 missing after pmd_offset returns the pmd. Thanks to the graceful
@@ -299,7 +299,7 @@ diff --git a/mm/mremap.c b/mm/mremap.c
                return NULL;
 
        pmd = pmd_offset(pud, addr);
-+      split_huge_page_pmd(mm, pmd);
++      split_huge_page_pmd(vma, addr, pmd);
        if (pmd_none_or_clear_bad(pmd))
                return NULL;
 

diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index 5c9687b1bde6706293e199331e7c866f1b752095..1dfe69cc78a81b90cc7d1edefc6d8e53f51c38c8 100644 (file)
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -182,7 +182,7 @@ static void mark_screen_rdonly(struct mm_struct *mm)
        if (pud_none_or_clear_bad(pud))
                goto out;
        pmd = pmd_offset(pud, 0xA0000);
-       split_huge_page_pmd(mm, pmd);
+       split_huge_page_pmd_mm(mm, 0xA0000, pmd);
        if (pmd_none_or_clear_bad(pmd))
                goto out;
        pte = pte_offset_map_lock(mm, pmd, 0xA0000, &ptl);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 90c63f9392a595318c5d1e03e1d69305bf9413fa..291a0d15a0be48fdc4792885cde2d9211e9acbef 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -643,7 +643,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
        spinlock_t *ptl;
        struct page *page;
 
-       split_huge_page_pmd(walk->mm, pmd);
+       split_huge_page_pmd(vma, addr, pmd);
        if (pmd_trans_unstable(pmd))
                return 0;
 

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 1af47755245944e9f25d794a1f18717960f49d46..3132ea788581d0aaf08b386bda28fee7684f4d81 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -95,12 +95,14 @@ extern int handle_pte_fault(struct mm_struct *mm,
                            struct vm_area_struct *vma, unsigned long address,
                            pte_t *pte, pmd_t *pmd, unsigned int flags);
 extern int split_huge_page(struct page *page);
-extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
-#define split_huge_page_pmd(__mm, __pmd)                               \
+extern void __split_huge_page_pmd(struct vm_area_struct *vma,
+               unsigned long address, pmd_t *pmd);
+#define split_huge_page_pmd(__vma, __address, __pmd)                   \
        do {                                                            \
                pmd_t *____pmd = (__pmd);                               \
                if (unlikely(pmd_trans_huge(*____pmd)))                 \
-                       __split_huge_page_pmd(__mm, ____pmd);           \
+                       __split_huge_page_pmd(__vma, __address,         \
+                                       ____pmd);                       \
        }  while (0)
 #define wait_split_huge_page(__anon_vma, __pmd)                                \
        do {                                                            \
@@ -110,6 +112,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
                BUG_ON(pmd_trans_splitting(*____pmd) ||                 \
                       pmd_trans_huge(*____pmd));                       \
        } while (0)
+extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
+               pmd_t *pmd);
 #if HPAGE_PMD_ORDER > MAX_ORDER
 #error "hugepages can't be allocated by the buddy allocator"
 #endif
@@ -177,10 +181,12 @@ static inline int split_huge_page(struct page *page)
 {
        return 0;
 }
-#define split_huge_page_pmd(__mm, __pmd)       \
+#define split_huge_page_pmd(__vma, __address, __pmd)   \
        do { } while (0)
 #define wait_split_huge_page(__anon_vma, __pmd)        \
        do { } while (0)
+#define split_huge_page_pmd_mm(__mm, __address, __pmd) \
+       do { } while (0)
 #define compound_trans_head(page) compound_head(page)
 static inline int hugepage_madvise(struct vm_area_struct *vma,
                                   unsigned long *vm_flags, int advice)

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 7742fb36eb4dd52e2d422eb3f50421b6a0c15928..de6aa5f3fdd272d949be177c6efb49c275d8a173 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2475,9 +2475,14 @@ static int khugepaged(void *none)
        return 0;
 }
 
-void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
+void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
+               pmd_t *pmd)
 {
        struct page *page;
+       unsigned long haddr = address & HPAGE_PMD_MASK;
+       struct mm_struct *mm = vma->vm_mm;
+
+       BUG_ON(vma->vm_start > haddr || vma->vm_end < haddr + HPAGE_PMD_SIZE);
 
        spin_lock(&mm->page_table_lock);
        if (unlikely(!pmd_trans_huge(*pmd))) {
@@ -2495,6 +2500,16 @@ void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
        BUG_ON(pmd_trans_huge(*pmd));
 }
 
+void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
+               pmd_t *pmd)
+{
+       struct vm_area_struct *vma;
+
+       vma = find_vma(mm, address);
+       BUG_ON(vma == NULL);
+       split_huge_page_pmd(vma, address, pmd);
+}
+
 static void split_huge_page_address(struct mm_struct *mm,
                                    unsigned long address)
 {
@@ -2509,7 +2524,7 @@ static void split_huge_page_address(struct mm_struct *mm,
         * Caller holds the mmap_sem write mode, so a huge pmd cannot
         * materialize from under us.
         */
-       split_huge_page_pmd(mm, pmd);
+       split_huge_page_pmd_mm(mm, address, pmd);
 }
 
 void __vma_adjust_trans_huge(struct vm_area_struct *vma,

diff --git a/mm/memory.c b/mm/memory.c
index 259b34fe1347632bf3ada80fd589730fca5e132f..f9a4b0cb862363792a127483a80d213017732610 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1243,7 +1243,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
                                        BUG();
                                }
 #endif
-                               split_huge_page_pmd(vma->vm_mm, pmd);
+                               split_huge_page_pmd(vma, addr, pmd);
                        } else if (zap_huge_pmd(tlb, vma, pmd, addr))
                                goto next;
                        /* fall through */
@@ -1512,7 +1512,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
        }
        if (pmd_trans_huge(*pmd)) {
                if (flags & FOLL_SPLIT) {
-                       split_huge_page_pmd(mm, pmd);
+                       split_huge_page_pmd(vma, address, pmd);
                        goto split_fallthrough;
                }
                spin_lock(&mm->page_table_lock);

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 05b28361a39b13b192aecd8545f73f6d607b2a2e..0719e8dd4945cb72b0d0c4a36ecb29f5d9784d2a 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -511,7 +511,7 @@ static inline int check_pmd_range(struct vm_area_struct *vma, pud_t *pud,
        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
-               split_huge_page_pmd(vma->vm_mm, pmd);
+               split_huge_page_pmd(vma, addr, pmd);
                if (pmd_none_or_trans_huge_or_clear_bad(pmd))
                        continue;
                if (check_pte_range(vma, pmd, addr, next, nodes,

diff --git a/mm/mprotect.c b/mm/mprotect.c
index a40992610ab6f6c0cbe96f9a9bd43a3fa973fd96..e8c3938db6faecf6f91c47db790a36271fa8d938 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -90,7 +90,7 @@ static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
                next = pmd_addr_end(addr, end);
                if (pmd_trans_huge(*pmd)) {
                        if (next - addr != HPAGE_PMD_SIZE)
-                               split_huge_page_pmd(vma->vm_mm, pmd);
+                               split_huge_page_pmd(vma, addr, pmd);
                        else if (change_huge_pmd(vma, pmd, addr, newprot))
                                continue;
                        /* fall through */

diff --git a/mm/mremap.c b/mm/mremap.c
index 1b61c2d3307a800c9c958e5f949d81fb76515899..eabb24da6c9e1bf00ee64075626630331ca94039 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -182,7 +182,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
                                need_flush = true;
                                continue;
                        } else if (!err) {
-                               split_huge_page_pmd(vma->vm_mm, old_pmd);
+                               split_huge_page_pmd(vma, old_addr, old_pmd);
                        }
                        VM_BUG_ON(pmd_trans_huge(*old_pmd));
                }

diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 6c118d012bb5a27be54ab331a4491e3ec421ac31..35aa294656cd812d2773fede0d51179a8b7ee09a 100644 (file)
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -58,7 +58,7 @@ again:
                if (!walk->pte_entry)
                        continue;
 
-               split_huge_page_pmd(walk->mm, pmd);
+               split_huge_page_pmd_mm(walk->mm, addr, pmd);
                if (pmd_none_or_trans_huge_or_clear_bad(pmd))
                        goto again;
                err = walk_pte_range(pmd, addr, next, walk);