s390/uaccess: simplify control register updates

Always switch to the kernel ASCE in switch_mm. Load the secondary
space ASCE in finish_arch_post_lock_switch after checking that
any pending page table operations have completed. The primary
ASCE is loaded in entry[64].S. With this the update_primary_asce
call can be removed from the switch_to macro and from the start
of switch_mm function. Remove the load_primary argument from
update_user_asce/clear_user_asce, rename update_user_asce to
set_user_asce and rename update_primary_asce to load_kernel_asce.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h
index 69cf5b5eddc95dcb83372e10f444df01825583c9..a4811aa0304d03264c4e63626d61619c62d16f0c 100644 (file)
--- a/arch/s390/include/asm/futex.h
+++ b/arch/s390/include/asm/futex.h
@@ -29,7 +29,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
        int cmparg = (encoded_op << 20) >> 20;
        int oldval = 0, newval, ret;
 
-       update_primary_asce(current);
+       load_kernel_asce();
        if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
                oparg = 1 << oparg;
 
@@ -79,7 +79,7 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 {
        int ret;
 
-       update_primary_asce(current);
+       load_kernel_asce();
        asm volatile(
                "   sacf 256\n"
                "0: cs   %1,%4,0(%5)\n"

diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 71be346d0e3c8074d7be6542815610567ff66606..93ec0c8e4c8342093e398818b52ca4fb02e63697 100644 (file)
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -30,33 +30,31 @@ static inline int init_new_context(struct task_struct *tsk,
 
 #define destroy_context(mm)             do { } while (0)
 
-static inline void update_user_asce(struct mm_struct *mm, int load_primary)
+static inline void set_user_asce(struct mm_struct *mm)
 {
        pgd_t *pgd = mm->pgd;
 
        S390_lowcore.user_asce = mm->context.asce_bits | __pa(pgd);
-       if (load_primary)
-               __ctl_load(S390_lowcore.user_asce, 1, 1);
        set_fs(current->thread.mm_segment);
+       set_thread_flag(TIF_ASCE);
 }
 
-static inline void clear_user_asce(struct mm_struct *mm, int load_primary)
+static inline void clear_user_asce(void)
 {
        S390_lowcore.user_asce = S390_lowcore.kernel_asce;
 
-       if (load_primary)
-               __ctl_load(S390_lowcore.user_asce, 1, 1);
+       __ctl_load(S390_lowcore.user_asce, 1, 1);
        __ctl_load(S390_lowcore.user_asce, 7, 7);
 }
 
-static inline void update_primary_asce(struct task_struct *tsk)
+static inline void load_kernel_asce(void)
 {
        unsigned long asce;
 
        __ctl_store(asce, 1, 1);
        if (asce != S390_lowcore.kernel_asce)
                __ctl_load(S390_lowcore.kernel_asce, 1, 1);
-       set_tsk_thread_flag(tsk, TIF_ASCE);
+       set_thread_flag(TIF_ASCE);
 }
 
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
@@ -64,25 +62,17 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 {
        int cpu = smp_processor_id();
 
-       update_primary_asce(tsk);
        if (prev == next)
                return;
        if (MACHINE_HAS_TLB_LC)
                cpumask_set_cpu(cpu, &next->context.cpu_attach_mask);
-       if (atomic_inc_return(&next->context.attach_count) >> 16) {
-               /* Delay update_user_asce until all TLB flushes are done. */
-               set_tsk_thread_flag(tsk, TIF_TLB_WAIT);
-               /* Clear old ASCE by loading the kernel ASCE. */
-               clear_user_asce(next, 0);
-       } else {
-               cpumask_set_cpu(cpu, mm_cpumask(next));
-               update_user_asce(next, 0);
-               if (next->context.flush_mm)
-                       /* Flush pending TLBs */
-                       __tlb_flush_mm(next);
-       }
+       /* Clear old ASCE by loading the kernel ASCE. */
+       __ctl_load(S390_lowcore.kernel_asce, 1, 1);
+       __ctl_load(S390_lowcore.kernel_asce, 7, 7);
+       /* Delay loading of the new ASCE to control registers CR1 & CR7 */
+       set_thread_flag(TIF_ASCE);
+       atomic_inc(&next->context.attach_count);
        atomic_dec(&prev->context.attach_count);
-       WARN_ON(atomic_read(&prev->context.attach_count) < 0);
        if (MACHINE_HAS_TLB_LC)
                cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask);
 }
@@ -93,15 +83,14 @@ static inline void finish_arch_post_lock_switch(void)
        struct task_struct *tsk = current;
        struct mm_struct *mm = tsk->mm;
 
-       if (!test_tsk_thread_flag(tsk, TIF_TLB_WAIT))
+       if (!mm)
                return;
        preempt_disable();
-       clear_tsk_thread_flag(tsk, TIF_TLB_WAIT);
        while (atomic_read(&mm->context.attach_count) >> 16)
                cpu_relax();
 
        cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
-       update_user_asce(mm, 0);
+       set_user_asce(mm);
        if (mm->context.flush_mm)
                __tlb_flush_mm(mm);
        preempt_enable();
@@ -113,7 +102,9 @@ static inline void finish_arch_post_lock_switch(void)
 static inline void activate_mm(struct mm_struct *prev,
                                struct mm_struct *next)
 {
-        switch_mm(prev, next, current);
+       switch_mm(prev, next, current);
+       cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
+       set_user_asce(next);
 }
 
 static inline void arch_dup_mmap(struct mm_struct *oldmm,

diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h
index e759181357fc5823c490696c23d047878c4ec753..29c81f82705e139dc53a9af3f72b0db3d9e14695 100644 (file)
--- a/arch/s390/include/asm/switch_to.h
+++ b/arch/s390/include/asm/switch_to.h
@@ -132,7 +132,6 @@ static inline void restore_access_regs(unsigned int *acrs)
                update_cr_regs(next);                                   \
        }                                                               \
        prev = __switch_to(prev,next);                                  \
-       update_primary_asce(current);                                   \
 } while (0)
 
 #define finish_arch_switch(prev) do {                                       \

diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h
index 50630e6a35de394688ac59207a5c523370493388..f3e5cad93b26b39fec25e55a192d2015212a6d86 100644 (file)
--- a/arch/s390/include/asm/thread_info.h
+++ b/arch/s390/include/asm/thread_info.h
@@ -81,8 +81,7 @@ static inline struct thread_info *current_thread_info(void)
 #define TIF_NOTIFY_RESUME      1       /* callback before returning to user */
 #define TIF_SIGPENDING         2       /* signal pending */
 #define TIF_NEED_RESCHED       3       /* rescheduling necessary */
-#define TIF_TLB_WAIT           4       /* wait for TLB flush completion */
-#define TIF_ASCE               5       /* primary asce needs fixup / uaccess */
+#define TIF_ASCE               5       /* user asce needs fixup / uaccess */
 #define TIF_PER_TRAP           6       /* deliver sigtrap on return to user */
 #define TIF_MCCK_PENDING       7       /* machine check handling is pending */
 #define TIF_SYSCALL_TRACE      8       /* syscall trace active */
@@ -99,7 +98,6 @@ static inline struct thread_info *current_thread_info(void)
 #define _TIF_NOTIFY_RESUME     (1<<TIF_NOTIFY_RESUME)
 #define _TIF_SIGPENDING                (1<<TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED      (1<<TIF_NEED_RESCHED)
-#define _TIF_TLB_WAIT          (1<<TIF_TLB_WAIT)
 #define _TIF_ASCE              (1<<TIF_ASCE)
 #define _TIF_PER_TRAP          (1<<TIF_PER_TRAP)
 #define _TIF_MCCK_PENDING      (1<<TIF_MCCK_PENDING)

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 1662038516c0db29d59a4a87dce89f374428cd51..7006bfdf5c5297f630c6b5bcc333655e221f95af 100644 (file)
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -43,7 +43,7 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
                 _TIF_MCCK_PENDING | _TIF_ASCE)
 _TIF_TRACE    = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
                 _TIF_SYSCALL_TRACEPOINT)
-_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT)
+_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_ASCE)
 
 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
 STACK_SIZE  = 1 << STACK_SHIFT

diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 5963e43618bb0df3ca790ffa1ce65a9fefd86b0b..d15e7bf6a863bc3f7a019397aec5966ac1b69025 100644 (file)
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -48,7 +48,7 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
                 _TIF_MCCK_PENDING | _TIF_ASCE)
 _TIF_TRACE    = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
                 _TIF_SYSCALL_TRACEPOINT)
-_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_TLB_WAIT)
+_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_ASCE)
 
 #define BASED(name) name-system_call(%r13)
 

diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index 7416efe8eae419c1249cf60c8f60dc268d1487cc..53dd5d7a0c964f4703562cefe915b0fb0317b554 100644 (file)
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -76,7 +76,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
 {
        unsigned long tmp1, tmp2;
 
-       update_primary_asce(current);
+       load_kernel_asce();
        tmp1 = -256UL;
        asm volatile(
                "   sacf  0\n"
@@ -159,7 +159,7 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
 {
        unsigned long tmp1, tmp2;
 
-       update_primary_asce(current);
+       load_kernel_asce();
        tmp1 = -256UL;
        asm volatile(
                "   sacf  0\n"
@@ -225,7 +225,7 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user
 {
        unsigned long tmp1;
 
-       update_primary_asce(current);
+       load_kernel_asce();
        asm volatile(
                "   sacf  256\n"
                "  "AHI"  %0,-1\n"
@@ -292,7 +292,7 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
 {
        unsigned long tmp1, tmp2;
 
-       update_primary_asce(current);
+       load_kernel_asce();
        asm volatile(
                "   sacf  256\n"
                "  "AHI"  %0,-1\n"
@@ -358,7 +358,7 @@ unsigned long __strnlen_user(const char __user *src, unsigned long size)
 {
        if (unlikely(!size))
                return 0;
-       update_primary_asce(current);
+       load_kernel_asce();
        return strnlen_user_srst(src, size);
 }
 EXPORT_SYMBOL(__strnlen_user);

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index d7cfd57815fbe484283819a98745f7e26e6d1d47..7881d4eb8b6bdf0891a89fa94b67bfebaab0a875 100644 (file)
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -53,8 +53,10 @@ static void __crst_table_upgrade(void *arg)
 {
        struct mm_struct *mm = arg;
 
-       if (current->active_mm == mm)
-               update_user_asce(mm, 1);
+       if (current->active_mm == mm) {
+               clear_user_asce();
+               set_user_asce(mm);
+       }
        __tlb_flush_local();
 }
 
@@ -108,7 +110,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
        pgd_t *pgd;
 
        if (current->active_mm == mm) {
-               clear_user_asce(mm, 1);
+               clear_user_asce();
                __tlb_flush_mm(mm);
        }
        while (mm->context.asce_limit > limit) {
@@ -134,7 +136,7 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
                crst_table_free(mm, (unsigned long *) pgd);
        }
        if (current->active_mm == mm)
-               update_user_asce(mm, 1);
+               set_user_asce(mm);
 }
 #endif