s390/mm: pmdp_get_and_clear_full optimization

Analog to ptep_get_and_clear_full define a variant of the
pmpd_get_and_clear primitive which gets the full hint from the
mmu_gather struct. This allows s390 to avoid a costly instruction
when destroying an address space.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 5ef1a266936ad4506b9582526815b53d9a86e43d..5e102422c9abd401ad999025d2c26e5b55d2a93f 100644 (file)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -1651,6 +1651,19 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
        return pmd;
 }
 
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR_FULL
+static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm,
+                                           unsigned long address,
+                                           pmd_t *pmdp, int full)
+{
+       pmd_t pmd = *pmdp;
+
+       if (!full)
+               pmdp_flush_lazy(mm, address, pmdp);
+       pmd_clear(pmdp);
+       return pmd;
+}
+
 #define __HAVE_ARCH_PMDP_CLEAR_FLUSH
 static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
                                     unsigned long address, pmd_t *pmdp)

diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index 752e30d639045c2ec970b479c680a08d4381d4e1..177d5973b132fb70ceba10684da7a600c9ec9dac 100644 (file)
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -103,6 +103,17 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 #endif
 
+#ifndef __HAVE_ARCH_PMDP_GET_AND_CLEAR_FULL
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm,
+                                           unsigned long address, pmd_t *pmdp,
+                                           int full)
+{
+       return pmdp_get_and_clear(mm, address, pmdp);
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#endif
+
 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
                                            unsigned long address, pte_t *ptep,

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 7e9c15cb93a9dd4a580d2d15a463f1030c941ff5..6a37f1b2ed1e5eecc1845a868b985a3092521d86 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1400,7 +1400,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 * pgtable_trans_huge_withdraw after finishing pmdp related
                 * operations.
                 */
-               orig_pmd = pmdp_get_and_clear(tlb->mm, addr, pmd);
+               orig_pmd = pmdp_get_and_clear_full(tlb->mm, addr, pmd,
+                                                  tlb->fullmm);
                tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
                pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
                if (is_huge_zero_pmd(orig_pmd)) {