thp: add tlb_remove_pmd_tlb_entry

We have tlb_remove_tlb_entry to indicate a pte tlb flush entry should be
flushed, but not a corresponding API for pmd entry.  This isn't a
problem so far because THP is only for x86 currently and tlb_flush()
under x86 will flush entire TLB.  But this is confusion and could be
missed if thp is ported to other arch.

Also convert tlb->need_flush = 1 to a VM_BUG_ON(!tlb->need_flush) in
__tlb_remove_page() as suggested by Andrea Arcangeli.  The
__tlb_remove_page() function is supposed to be called after
tlb_remove_xxx_tlb_entry() and we can catch any misuse.

Signed-off-by: Shaohua Li <shaohua.li@intel.com>
Reviewed-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index e58fa777fa09abe91831026852bfe21bdfa4f8d4..f96a5b58a975c6ccc1ebadecf746f84dd47a38ea 100644 (file)
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -139,6 +139,20 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
                __tlb_remove_tlb_entry(tlb, ptep, address);     \
        } while (0)
 
+/**
+ * tlb_remove_pmd_tlb_entry - remember a pmd mapping for later tlb invalidation
+ * This is a nop so far, because only x86 needs it.
+ */
+#ifndef __tlb_remove_pmd_tlb_entry
+#define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)
+#endif
+
+#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)           \
+       do {                                                    \
+               tlb->need_flush = 1;                            \
+               __tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
+       } while (0)
+
 #define pte_free_tlb(tlb, ptep, address)                       \
        do {                                                    \
                tlb->need_flush = 1;                            \

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index a9ace9c32507e31a851bcd719d0a56759b3a4428..1b921299abc420f1e258b614aa319964cd98942c 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -18,7 +18,7 @@ extern struct page *follow_trans_huge_pmd(struct mm_struct *mm,
                                          unsigned int flags);
 extern int zap_huge_pmd(struct mmu_gather *tlb,
                        struct vm_area_struct *vma,
-                       pmd_t *pmd);
+                       pmd_t *pmd, unsigned long addr);
 extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
                        unsigned long addr, unsigned long end,
                        unsigned char *vec);

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 964fc5a2edd2cfcabf83a0e4f8288ad59acab47d..5a595554bd8c842d930890fdf75168535fbec493 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1026,7 +1026,7 @@ out:
 }
 
 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
-                pmd_t *pmd)
+                pmd_t *pmd, unsigned long addr)
 {
        int ret = 0;
 
@@ -1042,6 +1042,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                        pgtable = get_pmd_huge_pte(tlb->mm);
                        page = pmd_page(*pmd);
                        pmd_clear(pmd);
+                       tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
                        page_remove_rmap(page);
                        VM_BUG_ON(page_mapcount(page) < 0);
                        add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);

diff --git a/mm/memory.c b/mm/memory.c
index 829d437354022959eeca37082f695e2ee73025ed..5e30583c2605d7b791d907ae8a5e07be2d9ad5b6 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -293,7 +293,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
        struct mmu_gather_batch *batch;
 
-       tlb->need_flush = 1;
+       VM_BUG_ON(!tlb->need_flush);
 
        if (tlb_fast_mode(tlb)) {
                free_page_and_swap_cache(page);
@@ -1231,7 +1231,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
                        if (next-addr != HPAGE_PMD_SIZE) {
                                VM_BUG_ON(!rwsem_is_locked(&tlb->mm->mmap_sem));
                                split_huge_page_pmd(vma->vm_mm, pmd);
-                       } else if (zap_huge_pmd(tlb, vma, pmd))
+                       } else if (zap_huge_pmd(tlb, vma, pmd, addr))
                                continue;
                        /* fall through */
                }