x86: Do not schedule while still in NMI context

The NMI handler uses the paranoid_exit routine that checks the
NEED_RESCHED flag, and if it is set and the return is for userspace,
then interrupts are enabled, the stack is swapped to the thread's stack,
and schedule is called. The problem with this is that we are still in an
NMI context until an iret is executed. This means that any new NMIs are
now starved until an interrupt or exception occurs and does the iret.

As NMIs can not be masked and can interrupt any location, they are
treated as a special case. NEED_RESCHED should not be set in an NMI
handler. The interruption by the NMI should not disturb the work flow
for scheduling. Any IPI sent to a processor after sending the
NEED_RESCHED would have to wait for the NMI anyway, and after the IPI
finishes the schedule would be called as required.

There is no reason to do anything special leaving an NMI. Remove the
call to paranoid_exit and do a simple return. This not only fixes the
bug of starved NMIs, but it also cleans up the code.

Link: http://lkml.kernel.org/r/CA+55aFzgM55hXTs4griX5e9=v_O+=ue+7Rj0PTD=M7hFYpyULQ@mail.gmail.com
Acked-by: Andi Kleen <ak@linux.intel.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: "H. Peter Anvin" <hpa@linux.intel.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Paul Turner <pjt@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index faf8d5e74b0be311198f74dabd88293587d06025..3819ea9073390ad13afe93711c7c51e4759a64d9 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1489,46 +1489,14 @@ ENTRY(nmi)
        movq %rsp,%rdi
        movq $-1,%rsi
        call do_nmi
-#ifdef CONFIG_TRACE_IRQFLAGS
-       /* paranoidexit; without TRACE_IRQS_OFF */
-       /* ebx: no swapgs flag */
-       DISABLE_INTERRUPTS(CLBR_NONE)
        testl %ebx,%ebx                         /* swapgs needed? */
        jnz nmi_restore
-       testl $3,CS(%rsp)
-       jnz nmi_userspace
 nmi_swapgs:
        SWAPGS_UNSAFE_STACK
 nmi_restore:
        RESTORE_ALL 8
        jmp irq_return
-nmi_userspace:
-       GET_THREAD_INFO(%rcx)
-       movl TI_flags(%rcx),%ebx
-       andl $_TIF_WORK_MASK,%ebx
-       jz nmi_swapgs
-       movq %rsp,%rdi                  /* &pt_regs */
-       call sync_regs
-       movq %rax,%rsp                  /* switch stack for scheduling */
-       testl $_TIF_NEED_RESCHED,%ebx
-       jnz nmi_schedule
-       movl %ebx,%edx                  /* arg3: thread flags */
-       ENABLE_INTERRUPTS(CLBR_NONE)
-       xorl %esi,%esi                  /* arg2: oldset */
-       movq %rsp,%rdi                  /* arg1: &pt_regs */
-       call do_notify_resume
-       DISABLE_INTERRUPTS(CLBR_NONE)
-       jmp nmi_userspace
-nmi_schedule:
-       ENABLE_INTERRUPTS(CLBR_ANY)
-       call schedule
-       DISABLE_INTERRUPTS(CLBR_ANY)
-       jmp nmi_userspace
        CFI_ENDPROC
-#else
-       jmp paranoid_exit
-       CFI_ENDPROC
-#endif
 END(nmi)
 
 ENTRY(ignore_sysret)