s390: run user space and KVM guests with modified branch prediction

[ Upstream commit 6b73044b2b0081ee3dd1cd6eaab7dee552601efb ]

Define TIF_ISOLATE_BP and TIF_ISOLATE_BP_GUEST and add the necessary
plumbing in entry.S to be able to run user space and KVM guests with
limited branch prediction.

To switch a user space process to limited branch prediction the
s390_isolate_bp() function has to be call, and to run a vCPU of a KVM
guest associated with the current task with limited branch prediction
call s390_isolate_bp_guest().

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 44de5d0014c54ebba8bd8f46695db215736a07e4..d5842126ec70ae5d2f3cab7ba720893ad8bc74ca 100644 (file)
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -360,6 +360,9 @@ extern void memcpy_absolute(void *, void *, size_t);
        memcpy_absolute(&(dest), &__tmp, sizeof(__tmp));        \
 }
 
+extern int s390_isolate_bp(void);
+extern int s390_isolate_bp_guest(void);
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __ASM_S390_PROCESSOR_H */

diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h
index f15c0398c3637ed49083ad4334c2b2b34e7e591c..84f2ae44b4e98ca1412ccfe5e60111cea60995f9 100644 (file)
--- a/arch/s390/include/asm/thread_info.h
+++ b/arch/s390/include/asm/thread_info.h
@@ -79,6 +79,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 #define TIF_SECCOMP            5       /* secure computing */
 #define TIF_SYSCALL_TRACEPOINT 6       /* syscall tracepoint instrumentation */
 #define TIF_UPROBE             7       /* breakpointed or single-stepping */
+#define TIF_ISOLATE_BP         8       /* Run process with isolated BP */
+#define TIF_ISOLATE_BP_GUEST   9       /* Run KVM guests with isolated BP */
 #define TIF_31BIT              16      /* 32bit process */
 #define TIF_MEMDIE             17      /* is terminating due to OOM killer */
 #define TIF_RESTORE_SIGMASK    18      /* restore signal mask in do_signal() */
@@ -94,6 +96,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 #define _TIF_SECCOMP           _BITUL(TIF_SECCOMP)
 #define _TIF_SYSCALL_TRACEPOINT        _BITUL(TIF_SYSCALL_TRACEPOINT)
 #define _TIF_UPROBE            _BITUL(TIF_UPROBE)
+#define _TIF_ISOLATE_BP                _BITUL(TIF_ISOLATE_BP)
+#define _TIF_ISOLATE_BP_GUEST  _BITUL(TIF_ISOLATE_BP_GUEST)
 #define _TIF_31BIT             _BITUL(TIF_31BIT)
 #define _TIF_SINGLE_STEP       _BITUL(TIF_SINGLE_STEP)
 

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index bfcf04de58086eb378b169fd54a5c37f33bde95e..0dd787aabac729d2f9ea141756fb2e8934676974 100644 (file)
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -105,6 +105,7 @@ _PIF_WORK   = (_PIF_PER_TRAP)
        j       3f
 1:     LAST_BREAK %r14
        UPDATE_VTIME %r14,%r15,\timer
+       BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 2:     lg      %r15,__LC_ASYNC_STACK   # load async stack
 3:     la      %r11,STACK_FRAME_OVERHEAD(%r15)
        .endm
@@ -191,6 +192,40 @@ _PIF_WORK  = (_PIF_PER_TRAP)
        .popsection
        .endm
 
+       .macro BPENTER tif_ptr,tif_mask
+       .pushsection .altinstr_replacement, "ax"
+662:   .word   0xc004, 0x0000, 0x0000  # 6 byte nop
+       .word   0xc004, 0x0000, 0x0000  # 6 byte nop
+       .popsection
+664:   TSTMSK  \tif_ptr,\tif_mask
+       jz      . + 8
+       .long   0xb2e8d000
+       .pushsection .altinstructions, "a"
+       .long 664b - .
+       .long 662b - .
+       .word 82
+       .byte 12
+       .byte 12
+       .popsection
+       .endm
+
+       .macro BPEXIT tif_ptr,tif_mask
+       TSTMSK  \tif_ptr,\tif_mask
+       .pushsection .altinstr_replacement, "ax"
+662:   jnz     . + 8
+       .long   0xb2e8d000
+       .popsection
+664:   jz      . + 8
+       .long   0xb2e8c000
+       .pushsection .altinstructions, "a"
+       .long 664b - .
+       .long 662b - .
+       .word 82
+       .byte 8
+       .byte 8
+       .popsection
+       .endm
+
        .section .kprobes.text, "ax"
 .Ldummy:
        /*
@@ -248,9 +283,11 @@ ENTRY(__switch_to)
  */
 ENTRY(sie64a)
        stmg    %r6,%r14,__SF_GPRS(%r15)        # save kernel registers
+       lg      %r12,__LC_CURRENT
        stg     %r2,__SF_EMPTY(%r15)            # save control block pointer
        stg     %r3,__SF_EMPTY+8(%r15)          # save guest register save area
        xc      __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
+       mvc     __SF_EMPTY+24(8,%r15),__TI_flags(%r12) # copy thread flags
        TSTMSK  __LC_CPU_FLAGS,_CIF_FPU         # load guest fp/vx registers ?
        jno     .Lsie_load_guest_gprs
        brasl   %r14,load_fpu_regs              # load guest fp/vx regs
@@ -267,10 +304,11 @@ ENTRY(sie64a)
        jnz     .Lsie_skip
        TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
        jo      .Lsie_skip                      # exit if fp/vx regs changed
-       BPON
+       BPEXIT  __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
        sie     0(%r14)
 .Lsie_exit:
        BPOFF
+       BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
 .Lsie_skip:
        ni      __SIE_PROG0C+3(%r14),0xfe       # no longer in SIE
        lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
@@ -332,6 +370,7 @@ ENTRY(system_call)
        LAST_BREAK %r13
 .Lsysc_vtime:
        UPDATE_VTIME %r10,%r13,__LC_SYNC_ENTER_TIMER
+       BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
        stmg    %r0,%r7,__PT_R0(%r11)
        # clear user controlled register to prevent speculative use
        xgr     %r0,%r0
@@ -369,7 +408,7 @@ ENTRY(system_call)
        jnz     .Lsysc_work                     # check for work
        TSTMSK  __LC_CPU_FLAGS,_CIF_WORK
        jnz     .Lsysc_work
-       BPON
+       BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
 .Lsysc_restore:
        lg      %r14,__LC_VDSO_PER_CPU
        lmg     %r0,%r10,__PT_R0(%r11)
@@ -555,6 +594,7 @@ ENTRY(pgm_check_handler)
        j       3f
 2:     LAST_BREAK %r14
        UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
+       BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
        lg      %r15,__LC_KERNEL_STACK
        lg      %r14,__TI_task(%r12)
        aghi    %r14,__TASK_thread      # pointer to thread_struct
@@ -683,7 +723,7 @@ ENTRY(io_int_handler)
        mvc     __LC_RETURN_PSW(16),__PT_PSW(%r11)
        tm      __PT_PSW+1(%r11),0x01   # returning to user ?
        jno     .Lio_exit_kernel
-       BPON
+       BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
 .Lio_exit_timer:
        stpt    __LC_EXIT_TIMER
        mvc     __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
@@ -1044,7 +1084,7 @@ ENTRY(mcck_int_handler)
        mvc     __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
        tm      __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
        jno     0f
-       BPON
+       BPEXIT  __TI_flags(%r12),_TIF_ISOLATE_BP
        stpt    __LC_EXIT_TIMER
        mvc     __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
 0:     lmg     %r11,%r15,__PT_R11(%r11)
@@ -1165,6 +1205,7 @@ cleanup_critical:
        .quad   .Lsie_done
 
 .Lcleanup_sie:
+       BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
        lg      %r9,__SF_EMPTY(%r15)            # get control block pointer
        ni      __SIE_PROG0C+3(%r9),0xfe        # no longer in SIE
        lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce

diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c
index 81d0808085e6bca9d2ff35c30619f7a238754db2..d856263fd7687eb5fcdf65da1fdef5bf25b1dfeb 100644 (file)
--- a/arch/s390/kernel/processor.c
+++ b/arch/s390/kernel/processor.c
@@ -179,3 +179,21 @@ const struct seq_operations cpuinfo_op = {
        .stop   = c_stop,
        .show   = show_cpuinfo,
 };
+
+int s390_isolate_bp(void)
+{
+       if (!test_facility(82))
+               return -EOPNOTSUPP;
+       set_thread_flag(TIF_ISOLATE_BP);
+       return 0;
+}
+EXPORT_SYMBOL(s390_isolate_bp);
+
+int s390_isolate_bp_guest(void)
+{
+       if (!test_facility(82))
+               return -EOPNOTSUPP;
+       set_thread_flag(TIF_ISOLATE_BP_GUEST);
+       return 0;
+}
+EXPORT_SYMBOL(s390_isolate_bp_guest);