x86/fpu: Rename fpu_save_init() to copy_fpregs_to_fpstate()

So fpu_save_init() is a historic name that got its name when the only
way the FPU state was FNSAVE, which cleared (well, destroyed) the FPU
state after saving it.

Nowadays the name is misleading, because ever since the introduction of
FXSAVE (and more modern FPU saving instructions) the 'we need to reload
the FPU state' part is only true if there's a pending FPU exception [*],
which is almost never the case.

So rename it to copy_fpregs_to_fpstate() to make it clear what's
happening. Also add a few comments about why we cannot keep registers
in certain cases.

Also clean up the control flow a bit, to make it more apparent when
we are dropping/keeping FP registers, and to optimize the common
case (of keeping fpregs) some more.

[*] Probably not true anymore, modern instructions always leave the FPU
    state intact, even if exceptions are pending: because pending FP
    exceptions are posted on the next FP instruction, not asynchronously.

    They were truly asynchronous back in the IRQ13 case, and we had to
    synchronize with them, but that code is not working anymore: we don't
    have IRQ13 mapped in the IDT anymore.

    But a cleanup patch is obviously not the place to change subtle behavior.

Reviewed-by: Borislav Petkov <bp@alien8.de>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h
index 89c6ec80c1ac757e7c6b151b914f952bf34458f2..11055f51e67ab10ef93a124cdd457d184d32a684 100644 (file)
--- a/arch/x86/include/asm/fpu/internal.h
+++ b/arch/x86/include/asm/fpu/internal.h
@@ -265,9 +265,15 @@ static inline void fpu_fxsave(struct fpu *fpu)
 
 /*
  * These must be called with preempt disabled. Returns
- * 'true' if the FPU state is still intact.
+ * 'true' if the FPU state is still intact and we can
+ * keep registers active.
+ *
+ * The legacy FNSAVE instruction cleared all FPU state
+ * unconditionally, so registers are essentially destroyed.
+ * Modern FPU state can be kept in registers, if there are
+ * no pending FP exceptions. (Note the FIXME below.)
  */
-static inline int fpu_save_init(struct fpu *fpu)
+static inline int copy_fpregs_to_fpstate(struct fpu *fpu)
 {
        if (use_xsave()) {
                xsave_state(&fpu->state->xsave);
@@ -276,13 +282,16 @@ static inline int fpu_save_init(struct fpu *fpu)
                 * xsave header may indicate the init state of the FP.
                 */
                if (!(fpu->state->xsave.header.xfeatures & XSTATE_FP))
-                       return 1;
-       } else if (use_fxsr()) {
-               fpu_fxsave(fpu);
+                       goto keep_fpregs;
        } else {
-               asm volatile("fnsave %[fx]; fwait"
-                            : [fx] "=m" (fpu->state->fsave));
-               return 0;
+               if (use_fxsr()) {
+                       fpu_fxsave(fpu);
+               } else {
+                       /* FNSAVE always clears FPU registers: */
+                       asm volatile("fnsave %[fx]; fwait"
+                                    : [fx] "=m" (fpu->state->fsave));
+                       goto drop_fpregs;
+               }
        }
 
        /*
@@ -295,9 +304,14 @@ static inline int fpu_save_init(struct fpu *fpu)
         */
        if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) {
                asm volatile("fnclex");
-               return 0;
+               goto drop_fpregs;
        }
+
+keep_fpregs:
        return 1;
+
+drop_fpregs:
+       return 0;
 }
 
 extern void fpu__save(struct fpu *fpu);
@@ -448,7 +462,7 @@ switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu)
                      (use_eager_fpu() || new_fpu->counter > 5);
 
        if (old_fpu->fpregs_active) {
-               if (!fpu_save_init(old_fpu))
+               if (!copy_fpregs_to_fpstate(old_fpu))
                        old_fpu->last_cpu = -1;
                else
                        old_fpu->last_cpu = cpu;

diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index b20d4ea8e132fa4af49e9f22bd5a66ec069a5070..ca88608a62a59a8c0f0f05dca74b20399e2af7ad 100644 (file)
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -102,7 +102,7 @@ void __kernel_fpu_begin(void)
        kernel_fpu_disable();
 
        if (fpu->fpregs_active) {
-               fpu_save_init(fpu);
+               copy_fpregs_to_fpstate(fpu);
        } else {
                this_cpu_write(fpu_fpregs_owner_ctx, NULL);
                if (!use_eager_fpu())
@@ -196,7 +196,7 @@ void fpu__save(struct fpu *fpu)
                if (use_eager_fpu()) {
                        __save_fpu(fpu);
                } else {
-                       fpu_save_init(fpu);
+                       copy_fpregs_to_fpstate(fpu);
                        fpregs_deactivate(fpu);
                }
        }

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index a65586edbb57073eb2b0ce06f9f79c2f3c1a4c89..f028f1da348025c0e9128535a467afc87a72fffe 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -395,7 +395,7 @@ dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
         * It is not directly accessible, though, so we need to
         * do an xsave and then pull it out of the xsave buffer.
         */
-       fpu_save_init(&tsk->thread.fpu);
+       copy_fpregs_to_fpstate(&tsk->thread.fpu);
        xsave_buf = &(tsk->thread.fpu.state->xsave);
        bndcsr = get_xsave_addr(xsave_buf, XSTATE_BNDCSR);
        if (!bndcsr)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 0b58b9397098d205183ea91c61cf5bca30cc463f..d90bf4afa2b06a30049f11c92750c14b1afa5851 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -7058,7 +7058,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
                return;
 
        vcpu->guest_fpu_loaded = 0;
-       fpu_save_init(&vcpu->arch.guest_fpu);
+       copy_fpregs_to_fpstate(&vcpu->arch.guest_fpu);
        __kernel_fpu_end();
        ++vcpu->stat.fpu_reload;
        kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);

diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c
index 5563be313fd6da005e3a15c144a4ed080c096e44..3287215be60a5d59951a6d2f3b4ed6b7acedfb7d 100644 (file)
--- a/arch/x86/mm/mpx.c
+++ b/arch/x86/mm/mpx.c
@@ -357,7 +357,7 @@ static __user void *task_get_bounds_dir(struct task_struct *tsk)
         * The bounds directory pointer is stored in a register
         * only accessible if we first do an xsave.
         */
-       fpu_save_init(&tsk->thread.fpu);
+       copy_fpregs_to_fpstate(&tsk->thread.fpu);
        bndcsr = get_xsave_addr(&tsk->thread.fpu.state->xsave, XSTATE_BNDCSR);
        if (!bndcsr)
                return MPX_INVALID_BOUNDS_DIR;