s390/mm: add guest ASCE TLB flush optimization

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index c92713893313e21ee59b102bf695b3fa0fda739e..b20b0f7170e82140146792b272fae0b2251e2cbc 100644 (file)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -953,9 +953,11 @@ static inline pte_t pte_mkhuge(pte_t pte)
 #define        IPTE_LOCAL      1
 
 #define IPTE_NODAT     0x400
+#define IPTE_GUEST_ASCE        0x800
 
 static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
-                              unsigned long opt, int local)
+                              unsigned long opt, unsigned long asce,
+                              int local)
 {
        unsigned long pto = (unsigned long) ptep;
 
@@ -969,6 +971,7 @@ static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
        }
 
        /* Invalidate ptes with options + TLB flush of the ptes */
+       opt = opt | (asce & _ASCE_ORIGIN);
        asm volatile(
                "       .insn   rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
                : [r2] "+a" (address), [r3] "+a" (opt)
@@ -1355,34 +1358,59 @@ static inline void __pmdp_csp(pmd_t *pmdp)
 
 #define IDTE_PTOA      0x0800
 #define IDTE_NODAT     0x1000
+#define IDTE_GUEST_ASCE        0x2000
 
 static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
-                              unsigned long opt, int local)
+                              unsigned long opt, unsigned long asce,
+                              int local)
 {
        unsigned long sto;
 
        sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);
-       asm volatile(
-               "       .insn   rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
-               : "+m" (*pmdp)
-               : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
-                 [m4] "i" (local)
-               : "cc" );
+       if (__builtin_constant_p(opt) && opt == 0) {
+               /* flush without guest asce */
+               asm volatile(
+                       "       .insn   rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
+                       : "+m" (*pmdp)
+                       : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
+                         [m4] "i" (local)
+                       : "cc" );
+       } else {
+               /* flush with guest asce */
+               asm volatile(
+                       "       .insn   rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
+                       : "+m" (*pmdp)
+                       : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
+                         [r3] "a" (asce), [m4] "i" (local)
+                       : "cc" );
+       }
 }
 
 static inline void __pudp_idte(unsigned long addr, pud_t *pudp,
-                              unsigned long opt, int local)
+                              unsigned long opt, unsigned long asce,
+                              int local)
 {
        unsigned long r3o;
 
        r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);
        r3o |= _ASCE_TYPE_REGION3;
-       asm volatile(
-               "       .insn   rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
-               : "+m" (*pudp)
-               : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
-                 [m4] "i" (local)
-               : "cc" );
+       if (__builtin_constant_p(opt) && opt == 0) {
+               /* flush without guest asce */
+               asm volatile(
+                       "       .insn   rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
+                       : "+m" (*pudp)
+                       : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
+                         [m4] "i" (local)
+                       : "cc");
+       } else {
+               /* flush with guest asce */
+               asm volatile(
+                       "       .insn   rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
+                       : "+m" (*pudp)
+                       : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
+                         [r3] "a" (asce), [m4] "i" (local)
+                       : "cc" );
+       }
 }
 
 pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);

diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h
index 38d82ed6034598ec2823960b60c6d3ec1fc28960..4d759f8f4bc737861849475e36d277adc653b2f0 100644 (file)
--- a/arch/s390/include/asm/tlbflush.h
+++ b/arch/s390/include/asm/tlbflush.h
@@ -23,6 +23,8 @@ static inline void __tlb_flush_idte(unsigned long asce)
        unsigned long opt;
 
        opt = IDTE_PTOA;
+       if (MACHINE_HAS_TLB_GUEST)
+               opt |= IDTE_GUEST_ASCE;
        /* Global TLB flush for the mm */
        asm volatile(
                "       .insn   rrf,0xb98e0000,0,%0,%1,0"

diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index 5734b01ca765e21844b4a1f9f22580c4f2f3b588..567fe92e2bb8fcdb29459d82b69ffbbd2d2e72fc 100644 (file)
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -328,7 +328,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr)
                return;
        }
        for (i = 0; i < nr; i++) {
-               __ptep_ipte(address, pte, 0, IPTE_GLOBAL);
+               __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL);
                address += PAGE_SIZE;
                pte++;
        }

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 9696bf89f03aacc398485377afdc3df1ed0c0002..3f1abc7b5fd228aad20aeb24dce24abf16e36feb 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -35,9 +35,13 @@ static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
                asce = READ_ONCE(mm->context.gmap_asce);
                if (asce == 0UL)
                        opt |= IPTE_NODAT;
-               __ptep_ipte(addr, ptep, opt, IPTE_LOCAL);
+               if (asce != -1UL) {
+                       asce = asce ? : mm->context.asce;
+                       opt |= IPTE_GUEST_ASCE;
+               }
+               __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
        } else {
-               __ptep_ipte(addr, ptep, 0, IPTE_LOCAL);
+               __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
        }
 }
 
@@ -51,9 +55,13 @@ static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
                asce = READ_ONCE(mm->context.gmap_asce);
                if (asce == 0UL)
                        opt |= IPTE_NODAT;
-               __ptep_ipte(addr, ptep, opt, IPTE_GLOBAL);
+               if (asce != -1UL) {
+                       asce = asce ? : mm->context.asce;
+                       opt |= IPTE_GUEST_ASCE;
+               }
+               __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
        } else {
-               __ptep_ipte(addr, ptep, 0, IPTE_GLOBAL);
+               __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
        }
 }
 
@@ -326,18 +334,20 @@ static inline void pmdp_idte_local(struct mm_struct *mm,
                                   unsigned long addr, pmd_t *pmdp)
 {
        if (MACHINE_HAS_TLB_GUEST)
-               __pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_LOCAL);
+               __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_LOCAL);
        else
-               __pmdp_idte(addr, pmdp, 0, IDTE_LOCAL);
+               __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
 }
 
 static inline void pmdp_idte_global(struct mm_struct *mm,
                                    unsigned long addr, pmd_t *pmdp)
 {
        if (MACHINE_HAS_TLB_GUEST)
-               __pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_GLOBAL);
+               __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_GLOBAL);
        else if (MACHINE_HAS_IDTE)
-               __pmdp_idte(addr, pmdp, 0, IDTE_GLOBAL);
+               __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
        else
                __pmdp_csp(pmdp);
 }
@@ -410,18 +420,20 @@ static inline void pudp_idte_local(struct mm_struct *mm,
                                   unsigned long addr, pud_t *pudp)
 {
        if (MACHINE_HAS_TLB_GUEST)
-               __pudp_idte(addr, pudp, IDTE_NODAT, IDTE_LOCAL);
+               __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_LOCAL);
        else
-               __pudp_idte(addr, pudp, 0, IDTE_LOCAL);
+               __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
 }
 
 static inline void pudp_idte_global(struct mm_struct *mm,
                                    unsigned long addr, pud_t *pudp)
 {
        if (MACHINE_HAS_TLB_GUEST)
-               __pudp_idte(addr, pudp, IDTE_NODAT, IDTE_GLOBAL);
+               __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
+                           mm->context.asce, IDTE_GLOBAL);
        else if (MACHINE_HAS_IDTE)
-               __pudp_idte(addr, pudp, 0, IDTE_GLOBAL);
+               __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
        else
                /*
                 * Invalid bit position is the same for pmd and pud, so we can