thp, s390: thp splitting backend for s390
authorGerald Schaefer <gerald.schaefer@de.ibm.com>
Mon, 8 Oct 2012 23:30:15 +0000 (16:30 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Tue, 9 Oct 2012 07:22:30 +0000 (16:22 +0900)
This patch is part of the architecture backend for thp on s390.  It
provides the functions related to thp splitting, including serialization
against gup.  Unlike other archs, pmdp_splitting_flush() cannot use a tlb
flushing operation to serialize against gup on s390, because that wouldn't
be stopped by the disabled IRQs.  So instead, smp_call_function() is
called with an empty function, which will have the expected effect.

Signed-off-by: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
arch/s390/include/asm/pgtable.h
arch/s390/mm/gup.c
arch/s390/mm/pgtable.c

index 6bd7d74830171096c82ac74f5dc671dbb7098090..c3f0775e71f1d3dead5be36d193ce99dc9c9a0e4 100644 (file)
@@ -347,6 +347,8 @@ extern struct page *vmemmap;
 
 #define _SEGMENT_ENTRY_LARGE   0x400   /* STE-format control, large page   */
 #define _SEGMENT_ENTRY_CO      0x100   /* change-recording override   */
+#define _SEGMENT_ENTRY_SPLIT_BIT 0     /* THP splitting bit number */
+#define _SEGMENT_ENTRY_SPLIT   (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
 
 /* Page status table bits for virtualization */
 #define RCP_ACC_BITS   0xf000000000000000UL
@@ -506,6 +508,10 @@ static inline int pmd_bad(pmd_t pmd)
        return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
 }
 
+#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
+extern void pmdp_splitting_flush(struct vm_area_struct *vma,
+                                unsigned long addr, pmd_t *pmdp);
+
 static inline int pte_none(pte_t pte)
 {
        return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
@@ -1159,6 +1165,13 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
 #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
 #define pte_unmap(pte) do { } while (0)
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline int pmd_trans_splitting(pmd_t pmd)
+{
+       return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
 /*
  * 31 bit swap entry format:
  * A page-table entry has some bits we have to treat in a special way.
index eeaf8023851f41e5c3516a7194cc3b7dbf3cba4a..60acb93a46809065b0d454e65f3837aa37e70aee 100644 (file)
@@ -115,7 +115,16 @@ static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
                pmd = *pmdp;
                barrier();
                next = pmd_addr_end(addr, end);
-               if (pmd_none(pmd))
+               /*
+                * The pmd_trans_splitting() check below explains why
+                * pmdp_splitting_flush() has to serialize with
+                * smp_call_function() against our disabled IRQs, to stop
+                * this gup-fast code from running while we set the
+                * splitting bit in the pmd. Returning zero will take
+                * the slow path that will call wait_split_huge_page()
+                * if the pmd is still in splitting state.
+                */
+               if (pmd_none(pmd) || pmd_trans_splitting(pmd))
                        return 0;
                if (unlikely(pmd_huge(pmd))) {
                        if (!gup_huge_pmd(pmdp, pmd, addr, next,
index b402991e43d71614cc111918a6b03a4af8525b2e..a6131d1fe6c0622b6ee443d4a7f0e255b2baae42 100644 (file)
@@ -866,3 +866,21 @@ bool kernel_page_present(struct page *page)
        return cc == 0;
 }
 #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static void pmdp_splitting_flush_sync(void *arg)
+{
+       /* Simply deliver the interrupt */
+}
+
+void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
+                         pmd_t *pmdp)
+{
+       VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+       if (!test_and_set_bit(_SEGMENT_ENTRY_SPLIT_BIT,
+                             (unsigned long *) pmdp)) {
+               /* need to serialize against gup-fast (IRQ disabled) */
+               smp_call_function(pmdp_splitting_flush_sync, NULL, 1);
+       }
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */