powerpc/mm/radix: Add tlb flush of THP ptes

Instead of flushing the entire mm, implement a flush_pmd_tlb_range

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 7ee9a195fa9c7b76f49836b07aeb547bae3254ae..30fad5d78e30cbb5873e0819d40f9091364d5a3e 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -12,6 +12,8 @@ static inline int mmu_get_ap(int psize)
 
 extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
                                         unsigned long end, int psize);
+extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+                                      unsigned long start, unsigned long end);
 extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
                            unsigned long end);
 extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);

diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 96e5769b18b00fe8fbcd7d788c3c94457d15a98a..0790c4e92a64520c55d472aa7d77195df8e746b7 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -7,6 +7,15 @@
 #include <asm/book3s/64/tlbflush-hash.h>
 #include <asm/book3s/64/tlbflush-radix.h>
 
+#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
+static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
+                                      unsigned long start, unsigned long end)
+{
+       if (radix_enabled())
+               return radix__flush_pmd_tlb_range(vma, start, end);
+       return hash__flush_tlb_range(vma, start, end);
+}
+
 static inline void flush_tlb_range(struct vm_area_struct *vma,
                                   unsigned long start, unsigned long end)
 {

diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index 670318766545c6fb029e4acfb7711be5ca7fe90d..7bb8acffe876b4f8dd1f8d2012a782a648c4cdb0 100644 (file)
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -33,7 +33,7 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
        changed = !pmd_same(*(pmdp), entry);
        if (changed) {
                __ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry));
-               flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+               flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        }
        return changed;
 }
@@ -66,7 +66,7 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
                     pmd_t *pmdp)
 {
        pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
-       flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+       flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        /*
         * This ensures that generic code that rely on IRQ disabling
         * to prevent a parallel THP split work as expected.

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 4d0f03ad099604ee9e5b659edd432ef9291221e4..456c155fe76c358fdbddec283a1f249446172008 100644 (file)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -391,3 +391,10 @@ void radix__flush_tlb_lpid(unsigned long lpid)
        asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 EXPORT_SYMBOL(radix__flush_tlb_lpid);
+
+void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+                               unsigned long start, unsigned long end)
+{
+       radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
+}
+EXPORT_SYMBOL(radix__flush_pmd_tlb_range);