x86/entry/32: Simplify and fix up the SYSENTER stack #DB/NMI fixup

Right after SYSENTER, we can get a #DB or NMI.  On x86_32, there's no IST,
so the exception handler is invoked on the temporary SYSENTER stack.

Because the SYSENTER stack is very small, we have a fixup to switch
off the stack quickly when this happens.  The old fixup had several issues:

 1. It checked the interrupt frame's CS and EIP.  This wasn't
    obviously correct on Xen or if vm86 mode was in use [1].

 2. In the NMI handler, it did some frightening digging into the
    stack frame.  I'm not convinced this digging was correct.

 3. The fixup didn't switch stacks and then switch back.  Instead, it
    synthesized a brand new stack frame that would redirect the IRET
    back to the SYSENTER code.  That frame was highly questionable.
    For one thing, if NMI nested inside #DB, we would effectively
    abort the #DB prologue, which was probably safe but was
    frightening.  For another, the code used PUSHFL to write the
    FLAGS portion of the frame, which was simply bogus -- by the time
    PUSHFL was called, at least TF, NT, VM, and all of the arithmetic
    flags were clobbered.

Simplify this considerably.  Instead of looking at the saved frame
to see where we came from, check the hardware ESP register against
the SYSENTER stack directly.  Malicious user code cannot spoof the
kernel ESP register, and by moving the check after SAVE_ALL, we can
use normal PER_CPU accesses to find all the relevant addresses.

With this patch applied, the improved syscall_nt_32 test finally
passes on 32-bit kernels.

[1] It isn't obviously correct, but it is nonetheless safe from vm86
    shenanigans as far as I can tell.  A user can't point EIP at
    entry_SYSENTER_32 while in vm86 mode because entry_SYSENTER_32,
    like all kernel addresses, is greater than 0xffff and would thus
    violate the CS segment limit.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Cc: Andrew Cooper <andrew.cooper3@citrix.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/b2cdbc037031c07ecf2c40a96069318aec0e7971.1457578375.git.luto@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index b2e1d446bdf8bac9d871f51af2c150434afdc36e..7b3ec24ede826a28ad272feaf880fc7babf6e66a 100644 (file)
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -976,51 +976,48 @@ error_code:
        jmp     ret_from_exception
 END(page_fault)
 
-/*
- * Debug traps and NMI can happen at the one SYSENTER instruction
- * that sets up the real kernel stack. Check here, since we can't
- * allow the wrong stack to be used.
- *
- * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
- * already pushed 3 words if it hits on the sysenter instruction:
- * eflags, cs and eip.
- *
- * We just load the right stack, and push the three (known) values
- * by hand onto the new stack - while updating the return eip past
- * the instruction that would have done it for sysenter.
- */
-.macro FIX_STACK offset ok label
-       cmpw    $__KERNEL_CS, 4(%esp)
-       jne     \ok
-\label:
-       movl    TSS_sysenter_sp0 + \offset(%esp), %esp
-       pushfl
-       pushl   $__KERNEL_CS
-       pushl   $sysenter_past_esp
-.endm
-
 ENTRY(debug)
+       /*
+        * #DB can happen at the first instruction of
+        * entry_SYSENTER_32 or in Xen's SYSENTER prologue.  If this
+        * happens, then we will be running on a very small stack.  We
+        * need to detect this condition and switch to the thread
+        * stack before calling any C code at all.
+        *
+        * If you edit this code, keep in mind that NMIs can happen in here.
+        */
        ASM_CLAC
-       cmpl    $entry_SYSENTER_32, (%esp)
-       jne     debug_stack_correct
-       FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
-debug_stack_correct:
        pushl   $-1                             # mark this as an int
        SAVE_ALL
-       TRACE_IRQS_OFF
        xorl    %edx, %edx                      # error code 0
        movl    %esp, %eax                      # pt_regs pointer
+
+       /* Are we currently on the SYSENTER stack? */
+       PER_CPU(cpu_tss + CPU_TSS_SYSENTER_stack + SIZEOF_SYSENTER_stack, %ecx)
+       subl    %eax, %ecx      /* ecx = (end of SYSENTER_stack) - esp */
+       cmpl    $SIZEOF_SYSENTER_stack, %ecx
+       jb      .Ldebug_from_sysenter_stack
+
+       TRACE_IRQS_OFF
+       call    do_debug
+       jmp     ret_from_exception
+
+.Ldebug_from_sysenter_stack:
+       /* We're on the SYSENTER stack.  Switch off. */
+       movl    %esp, %ebp
+       movl    PER_CPU_VAR(cpu_current_top_of_stack), %esp
+       TRACE_IRQS_OFF
        call    do_debug
+       movl    %ebp, %esp
        jmp     ret_from_exception
 END(debug)
 
 /*
- * NMI is doubly nasty. It can happen _while_ we're handling
- * a debug fault, and the debug fault hasn't yet been able to
- * clear up the stack. So we first check whether we got  an
- * NMI on the sysenter entry path, but after that we need to
- * check whether we got an NMI on the debug path where the debug
- * fault happened on the sysenter path.
+ * NMI is doubly nasty.  It can happen on the first instruction of
+ * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
+ * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
+ * switched stacks.  We handle both conditions by simply checking whether we
+ * interrupted kernel code running on the SYSENTER stack.
  */
 ENTRY(nmi)
        ASM_CLAC
@@ -1031,41 +1028,32 @@ ENTRY(nmi)
        popl    %eax
        je      nmi_espfix_stack
 #endif
-       cmpl    $entry_SYSENTER_32, (%esp)
-       je      nmi_stack_fixup
-       pushl   %eax
-       movl    %esp, %eax
-       /*
-        * Do not access memory above the end of our stack page,
-        * it might not exist.
-        */
-       andl    $(THREAD_SIZE-1), %eax
-       cmpl    $(THREAD_SIZE-20), %eax
-       popl    %eax
-       jae     nmi_stack_correct
-       cmpl    $entry_SYSENTER_32, 12(%esp)
-       je      nmi_debug_stack_check
-nmi_stack_correct:
-       pushl   %eax
+
+       pushl   %eax                            # pt_regs->orig_ax
        SAVE_ALL
        xorl    %edx, %edx                      # zero error code
        movl    %esp, %eax                      # pt_regs pointer
+
+       /* Are we currently on the SYSENTER stack? */
+       PER_CPU(cpu_tss + CPU_TSS_SYSENTER_stack + SIZEOF_SYSENTER_stack, %ecx)
+       subl    %eax, %ecx      /* ecx = (end of SYSENTER_stack) - esp */
+       cmpl    $SIZEOF_SYSENTER_stack, %ecx
+       jb      .Lnmi_from_sysenter_stack
+
+       /* Not on SYSENTER stack. */
        call    do_nmi
        jmp     restore_all_notrace
 
-nmi_stack_fixup:
-       FIX_STACK 12, nmi_stack_correct, 1
-       jmp     nmi_stack_correct
-
-nmi_debug_stack_check:
-       cmpw    $__KERNEL_CS, 16(%esp)
-       jne     nmi_stack_correct
-       cmpl    $debug, (%esp)
-       jb      nmi_stack_correct
-       cmpl    $debug_esp_fix_insn, (%esp)
-       ja      nmi_stack_correct
-       FIX_STACK 24, nmi_stack_correct, 1
-       jmp     nmi_stack_correct
+.Lnmi_from_sysenter_stack:
+       /*
+        * We're on the SYSENTER stack.  Switch off.  No one (not even debug)
+        * is using the thread stack right now, so it's safe for us to use it.
+        */
+       movl    %esp, %ebp
+       movl    PER_CPU_VAR(cpu_current_top_of_stack), %esp
+       call    do_nmi
+       movl    %ebp, %esp
+       jmp     restore_all_notrace
 
 #ifdef CONFIG_X86_ESPFIX32
 nmi_espfix_stack:

diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index fdeb0ce07c166d38c7cdb5639a6e173932450c13..ecdc1d217dc0f50a7b760c25cec8bee586643336 100644 (file)
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -52,6 +52,11 @@ void foo(void)
        DEFINE(TSS_sysenter_sp0, offsetof(struct tss_struct, x86_tss.sp0) -
               offsetofend(struct tss_struct, SYSENTER_stack));
 
+       /* Offset from cpu_tss to SYSENTER_stack */
+       OFFSET(CPU_TSS_SYSENTER_stack, tss_struct, SYSENTER_stack);
+       /* Size of SYSENTER_stack */
+       DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
+
 #if defined(CONFIG_LGUEST) || defined(CONFIG_LGUEST_GUEST) || defined(CONFIG_LGUEST_MODULE)
        BLANK();
        OFFSET(LGUEST_DATA_irq_enabled, lguest_data, irq_enabled);