s390/mm: make use of ipte range facility

Invalidate several pte entries at once if the ipte range facility
is available. Currently this works only for DEBUG_PAGE_ALLOC where
several up to 2 ^ MAX_ORDER may be invalidated at once.

Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 7c4af56f2b7392ef1f6a7f4f2f8f7b56baf7b3d6..2de229c1b467ec326bf850750852212d21704f03 100644 (file)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -1053,6 +1053,22 @@ static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep)
                : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address));
 }
 
+static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep)
+{
+       unsigned long pto = (unsigned long) ptep;
+
+#ifndef CONFIG_64BIT
+       /* pto in ESA mode must point to the start of the segment table */
+       pto &= 0x7ffffc00;
+#endif
+       /* Invalidate a range of ptes + global TLB flush of the ptes */
+       do {
+               asm volatile(
+                       "       .insn rrf,0xb2210000,%2,%0,%1,0"
+                       : "+a" (address), "+a" (nr) : "a" (pto) : "memory");
+       } while (nr != 255);
+}
+
 static inline void ptep_flush_direct(struct mm_struct *mm,
                                     unsigned long address, pte_t *ptep)
 {

diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c
index 8400f494623f4591a5de3e5ea6442a08fe767ed4..3fef3b299665797bbd486baec5239d5f88705428 100644 (file)
--- a/arch/s390/mm/pageattr.c
+++ b/arch/s390/mm/pageattr.c
@@ -6,6 +6,7 @@
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <asm/cacheflush.h>
+#include <asm/facility.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
 
@@ -103,27 +104,50 @@ int set_memory_x(unsigned long addr, int numpages)
 }
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
+
+static void ipte_range(pte_t *pte, unsigned long address, int nr)
+{
+       int i;
+
+       if (test_facility(13) && IS_ENABLED(CONFIG_64BIT)) {
+               __ptep_ipte_range(address, nr - 1, pte);
+               return;
+       }
+       for (i = 0; i < nr; i++) {
+               __ptep_ipte(address, pte);
+               address += PAGE_SIZE;
+               pte++;
+       }
+}
+
 void kernel_map_pages(struct page *page, int numpages, int enable)
 {
        unsigned long address;
+       int nr, i, j;
        pgd_t *pgd;
        pud_t *pud;
        pmd_t *pmd;
        pte_t *pte;
-       int i;
 
-       for (i = 0; i < numpages; i++) {
+       for (i = 0; i < numpages;) {
                address = page_to_phys(page + i);
                pgd = pgd_offset_k(address);
                pud = pud_offset(pgd, address);
                pmd = pmd_offset(pud, address);
                pte = pte_offset_kernel(pmd, address);
-               if (!enable) {
-                       __ptep_ipte(address, pte);
-                       pte_val(*pte) = _PAGE_INVALID;
-                       continue;
+               nr = (unsigned long)pte >> ilog2(sizeof(long));
+               nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1));
+               nr = min(numpages - i, nr);
+               if (enable) {
+                       for (j = 0; j < nr; j++) {
+                               pte_val(*pte) = __pa(address);
+                               address += PAGE_SIZE;
+                               pte++;
+                       }
+               } else {
+                       ipte_range(pte, address, nr);
                }
-               pte_val(*pte) = __pa(address);
+               i += nr;
        }
 }