thp, s390: architecture backend for thp on s390
authorGerald Schaefer <gerald.schaefer@de.ibm.com>
Mon, 8 Oct 2012 23:30:24 +0000 (16:30 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Tue, 9 Oct 2012 07:22:31 +0000 (16:22 +0900)
This implements the architecture backend for transparent hugepages
on s390.

Signed-off-by: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
arch/s390/include/asm/hugetlb.h
arch/s390/include/asm/pgtable.h
arch/s390/include/asm/setup.h
arch/s390/include/asm/tlb.h
arch/s390/kernel/early.c
arch/s390/mm/pgtable.c

index fc322421b1cc127dcb640407454c8e01bbb00df8..593753ee07f3c55126f5910f4fa4d4992d6a14de 100644 (file)
@@ -78,23 +78,6 @@ static inline void __pmd_csp(pmd_t *pmdp)
                "       csp %1,%3"
                : "=m" (*pmdp)
                : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc");
-       pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
-}
-
-static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
-{
-       unsigned long sto = (unsigned long) pmdp -
-                               pmd_index(address) * sizeof(pmd_t);
-
-       if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
-               asm volatile(
-                       "       .insn   rrf,0xb98e0000,%2,%3,0,0"
-                       : "=m" (*pmdp)
-                       : "m" (*pmdp), "a" (sto),
-                         "a" ((address & HPAGE_MASK))
-               );
-       }
-       pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
 }
 
 static inline void huge_ptep_invalidate(struct mm_struct *mm,
@@ -106,6 +89,7 @@ static inline void huge_ptep_invalidate(struct mm_struct *mm,
                __pmd_idte(address, pmdp);
        else
                __pmd_csp(pmdp);
+       pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
 }
 
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
index 2c8f00d31cfe27476d4e2ba1d3ac20b6db43bf95..ed14fc2db6e0e98abf5805760eb92602bb5a3547 100644 (file)
@@ -350,6 +350,10 @@ extern struct page *vmemmap;
 #define _SEGMENT_ENTRY_SPLIT_BIT 0     /* THP splitting bit number */
 #define _SEGMENT_ENTRY_SPLIT   (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
 
+/* Set of bits not changed in pmd_modify */
+#define _SEGMENT_CHG_MASK      (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
+                                | _SEGMENT_ENTRY_SPLIT | _SEGMENT_ENTRY_CO)
+
 /* Page status table bits for virtualization */
 #define RCP_ACC_BITS   0xf000000000000000UL
 #define RCP_FP_BIT     0x0800000000000000UL
@@ -512,6 +516,26 @@ static inline int pmd_bad(pmd_t pmd)
 extern void pmdp_splitting_flush(struct vm_area_struct *vma,
                                 unsigned long addr, pmd_t *pmdp);
 
+#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
+extern int pmdp_set_access_flags(struct vm_area_struct *vma,
+                                unsigned long address, pmd_t *pmdp,
+                                pmd_t entry, int dirty);
+
+#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
+extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
+                                 unsigned long address, pmd_t *pmdp);
+
+#define __HAVE_ARCH_PMD_WRITE
+static inline int pmd_write(pmd_t pmd)
+{
+       return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0;
+}
+
+static inline int pmd_young(pmd_t pmd)
+{
+       return 0;
+}
+
 static inline int pte_none(pte_t pte)
 {
        return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
@@ -1165,6 +1189,22 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
 #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
 #define pte_unmap(pte) do { } while (0)
 
+static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
+{
+       unsigned long sto = (unsigned long) pmdp -
+                           pmd_index(address) * sizeof(pmd_t);
+
+       if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
+               asm volatile(
+                       "       .insn   rrf,0xb98e0000,%2,%3,0,0"
+                       : "=m" (*pmdp)
+                       : "m" (*pmdp), "a" (sto),
+                         "a" ((address & HPAGE_MASK))
+                       : "cc"
+               );
+       }
+}
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define __HAVE_ARCH_PGTABLE_DEPOSIT
 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable);
@@ -1176,6 +1216,156 @@ static inline int pmd_trans_splitting(pmd_t pmd)
 {
        return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
 }
+
+static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+                             pmd_t *pmdp, pmd_t entry)
+{
+       *pmdp = entry;
+}
+
+static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
+{
+       unsigned long pgprot_pmd = 0;
+
+       if (pgprot_val(pgprot) & _PAGE_INVALID) {
+               if (pgprot_val(pgprot) & _PAGE_SWT)
+                       pgprot_pmd |= _HPAGE_TYPE_NONE;
+               pgprot_pmd |= _SEGMENT_ENTRY_INV;
+       }
+       if (pgprot_val(pgprot) & _PAGE_RO)
+               pgprot_pmd |= _SEGMENT_ENTRY_RO;
+       return pgprot_pmd;
+}
+
+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+       pmd_val(pmd) &= _SEGMENT_CHG_MASK;
+       pmd_val(pmd) |= massage_pgprot_pmd(newprot);
+       return pmd;
+}
+
+static inline pmd_t pmd_mkhuge(pmd_t pmd)
+{
+       pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
+       return pmd;
+}
+
+static inline pmd_t pmd_mkwrite(pmd_t pmd)
+{
+       pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO;
+       return pmd;
+}
+
+static inline pmd_t pmd_wrprotect(pmd_t pmd)
+{
+       pmd_val(pmd) |= _SEGMENT_ENTRY_RO;
+       return pmd;
+}
+
+static inline pmd_t pmd_mkdirty(pmd_t pmd)
+{
+       /* No dirty bit in the segment table entry. */
+       return pmd;
+}
+
+static inline pmd_t pmd_mkold(pmd_t pmd)
+{
+       /* No referenced bit in the segment table entry. */
+       return pmd;
+}
+
+static inline pmd_t pmd_mkyoung(pmd_t pmd)
+{
+       /* No referenced bit in the segment table entry. */
+       return pmd;
+}
+
+#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
+static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+                                           unsigned long address, pmd_t *pmdp)
+{
+       unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK;
+       long tmp, rc;
+       int counter;
+
+       rc = 0;
+       if (MACHINE_HAS_RRBM) {
+               counter = PTRS_PER_PTE >> 6;
+               asm volatile(
+                       "0:     .insn   rre,0xb9ae0000,%0,%3\n" /* rrbm */
+                       "       ogr     %1,%0\n"
+                       "       la      %3,0(%4,%3)\n"
+                       "       brct    %2,0b\n"
+                       : "=&d" (tmp), "+&d" (rc), "+d" (counter),
+                         "+a" (pmd_addr)
+                       : "a" (64 * 4096UL) : "cc");
+               rc = !!rc;
+       } else {
+               counter = PTRS_PER_PTE;
+               asm volatile(
+                       "0:     rrbe    0,%2\n"
+                       "       la      %2,0(%3,%2)\n"
+                       "       brc     12,1f\n"
+                       "       lhi     %0,1\n"
+                       "1:     brct    %1,0b\n"
+                       : "+d" (rc), "+d" (counter), "+a" (pmd_addr)
+                       : "a" (4096UL) : "cc");
+       }
+       return rc;
+}
+
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
+static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
+                                      unsigned long address, pmd_t *pmdp)
+{
+       pmd_t pmd = *pmdp;
+
+       __pmd_idte(address, pmdp);
+       pmd_clear(pmdp);
+       return pmd;
+}
+
+#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
+static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
+                                    unsigned long address, pmd_t *pmdp)
+{
+       return pmdp_get_and_clear(vma->vm_mm, address, pmdp);
+}
+
+#define __HAVE_ARCH_PMDP_INVALIDATE
+static inline void pmdp_invalidate(struct vm_area_struct *vma,
+                                  unsigned long address, pmd_t *pmdp)
+{
+       __pmd_idte(address, pmdp);
+}
+
+static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
+{
+       pmd_t __pmd;
+       pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
+       return __pmd;
+}
+
+#define pfn_pmd(pfn, pgprot)   mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
+#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
+
+static inline int pmd_trans_huge(pmd_t pmd)
+{
+       return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
+}
+
+static inline int has_transparent_hugepage(void)
+{
+       return MACHINE_HAS_HPAGE ? 1 : 0;
+}
+
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+       if (pmd_trans_huge(pmd))
+               return pmd_val(pmd) >> HPAGE_SHIFT;
+       else
+               return pmd_val(pmd) >> PAGE_SHIFT;
+}
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 /*
index 87b47ca954f1c1ceec09ee083427eae697166242..8cfd731a18d86260dfbf0cb025cc24b19ff997a4 100644 (file)
@@ -81,6 +81,7 @@ extern unsigned int s390_user_mode;
 #define MACHINE_FLAG_SPP       (1UL << 13)
 #define MACHINE_FLAG_TOPOLOGY  (1UL << 14)
 #define MACHINE_FLAG_TE                (1UL << 15)
+#define MACHINE_FLAG_RRBM      (1UL << 16)
 
 #define MACHINE_IS_VM          (S390_lowcore.machine_flags & MACHINE_FLAG_VM)
 #define MACHINE_IS_KVM         (S390_lowcore.machine_flags & MACHINE_FLAG_KVM)
@@ -99,7 +100,8 @@ extern unsigned int s390_user_mode;
 #define MACHINE_HAS_PFMF       (0)
 #define MACHINE_HAS_SPP                (0)
 #define MACHINE_HAS_TOPOLOGY   (0)
-#define MACHINE_HAS_TE               (0)
+#define MACHINE_HAS_TE         (0)
+#define MACHINE_HAS_RRBM       (0)
 #else /* CONFIG_64BIT */
 #define MACHINE_HAS_IEEE       (1)
 #define MACHINE_HAS_CSP                (1)
@@ -112,6 +114,7 @@ extern unsigned int s390_user_mode;
 #define MACHINE_HAS_SPP                (S390_lowcore.machine_flags & MACHINE_FLAG_SPP)
 #define MACHINE_HAS_TOPOLOGY   (S390_lowcore.machine_flags & MACHINE_FLAG_TOPOLOGY)
 #define MACHINE_HAS_TE         (S390_lowcore.machine_flags & MACHINE_FLAG_TE)
+#define MACHINE_HAS_RRBM       (S390_lowcore.machine_flags & MACHINE_FLAG_RRBM)
 #endif /* CONFIG_64BIT */
 
 #define ZFCPDUMP_HSA_SIZE      (32UL<<20)
index 06e5acbc84bd50ef4917eabb5f6fc8cd2d6f6b22..b75d7d686684975278e1881a9aaf74590a6949ed 100644 (file)
@@ -137,6 +137,7 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 #define tlb_start_vma(tlb, vma)                        do { } while (0)
 #define tlb_end_vma(tlb, vma)                  do { } while (0)
 #define tlb_remove_tlb_entry(tlb, ptep, addr)  do { } while (0)
+#define tlb_remove_pmd_tlb_entry(tlb, pmdp, addr)      do { } while (0)
 #define tlb_migrate_finish(mm)                 do { } while (0)
 
 #endif /* _S390_TLB_H */
index 7f4717675c199ce53ad0a1580bcc25c83dd9442c..00d11444506882d9cb771dd0cd2874902c549727 100644 (file)
@@ -388,6 +388,8 @@ static __init void detect_machine_facilities(void)
                S390_lowcore.machine_flags |= MACHINE_FLAG_SPP;
        if (test_facility(50) && test_facility(73))
                S390_lowcore.machine_flags |= MACHINE_FLAG_TE;
+       if (test_facility(66))
+               S390_lowcore.machine_flags |= MACHINE_FLAG_RRBM;
 #endif
 }
 
index e86a523875eb8d2cf4a0386ae6e48fd4bef291c2..c8188a18af05b565e57304b02e153ed4629940d5 100644 (file)
@@ -898,6 +898,28 @@ bool kernel_page_present(struct page *page)
 #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
+                          pmd_t *pmdp)
+{
+       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+       /* No need to flush TLB
+        * On s390 reference bits are in storage key and never in TLB */
+       return pmdp_test_and_clear_young(vma, address, pmdp);
+}
+
+int pmdp_set_access_flags(struct vm_area_struct *vma,
+                         unsigned long address, pmd_t *pmdp,
+                         pmd_t entry, int dirty)
+{
+       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+       if (pmd_same(*pmdp, entry))
+               return 0;
+       pmdp_invalidate(vma, address, pmdp);
+       set_pmd_at(vma->vm_mm, address, pmdp, entry);
+       return 1;
+}
+
 static void pmdp_splitting_flush_sync(void *arg)
 {
        /* Simply deliver the interrupt */