x86, fpu: decouple non-lazy/eager fpu restore from xsave
authorSuresh Siddha <suresh.b.siddha@intel.com>
Thu, 6 Sep 2012 21:58:52 +0000 (14:58 -0700)
committerH. Peter Anvin <hpa@linux.intel.com>
Tue, 18 Sep 2012 22:52:22 +0000 (15:52 -0700)
Decouple non-lazy/eager fpu restore policy from the existence of the xsave
feature. Introduce a synthetic CPUID flag to represent the eagerfpu
policy. "eagerfpu=on" boot paramter will enable the policy.

Requested-by: H. Peter Anvin <hpa@zytor.com>
Requested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
Link: http://lkml.kernel.org/r/1347300665-6209-2-git-send-email-suresh.b.siddha@intel.com
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Documentation/kernel-parameters.txt
arch/x86/include/asm/cpufeature.h
arch/x86/include/asm/fpu-internal.h
arch/x86/kernel/cpu/common.c
arch/x86/kernel/i387.c
arch/x86/kernel/process.c
arch/x86/kernel/traps.c
arch/x86/kernel/xsave.c

index ad7e2e5088c126ce48e0362d1f3296f3e8b55d2b..741d064fdc6acac7357b1f5831a041764146d39e 100644 (file)
@@ -1833,6 +1833,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
                        and restore using xsave. The kernel will fallback to
                        enabling legacy floating-point and sse state.
 
+       eagerfpu=       [X86]
+                       on      enable eager fpu restore
+                       off     disable eager fpu restore
+
        nohlt           [BUGS=ARM,SH] Tells the kernel that the sleep(SH) or
                        wfi(ARM) instruction doesn't work correctly and not to
                        use it. This is also useful when using JTAG debugger.
index 6b7ee5ff682024f80ec205bf7bdc9c7814787e01..5dd2b473ccffdfba3a7e9d8794a496a6be05311a 100644 (file)
@@ -97,6 +97,7 @@
 #define X86_FEATURE_EXTD_APICID        (3*32+26) /* has extended APICID (8 bits) */
 #define X86_FEATURE_AMD_DCM     (3*32+27) /* multi-node processor */
 #define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */
+#define X86_FEATURE_EAGER_FPU  (3*32+29) /* "eagerfpu" Non lazy FPU restore */
 
 /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
 #define X86_FEATURE_XMM3       (4*32+ 0) /* "pni" SSE-3 */
@@ -305,6 +306,7 @@ extern const char * const x86_power_flags[32];
 #define cpu_has_perfctr_core   boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
 #define cpu_has_cx8            boot_cpu_has(X86_FEATURE_CX8)
 #define cpu_has_cx16           boot_cpu_has(X86_FEATURE_CX16)
+#define cpu_has_eager_fpu      boot_cpu_has(X86_FEATURE_EAGER_FPU)
 
 #if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
 # define cpu_has_invlpg                1
index 8ca0f9f45ac49d95036198a679bed9522f7d1e70..0ca72f0d4b417425565c1e0593537c10f54e77e0 100644 (file)
@@ -38,6 +38,7 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
 
 extern unsigned int mxcsr_feature_mask;
 extern void fpu_init(void);
+extern void eager_fpu_init(void);
 
 DECLARE_PER_CPU(struct task_struct *, fpu_owner_task);
 
@@ -84,6 +85,11 @@ static inline int is_x32_frame(void)
 
 #define X87_FSW_ES (1 << 7)    /* Exception Summary */
 
+static __always_inline __pure bool use_eager_fpu(void)
+{
+       return static_cpu_has(X86_FEATURE_EAGER_FPU);
+}
+
 static __always_inline __pure bool use_xsaveopt(void)
 {
        return static_cpu_has(X86_FEATURE_XSAVEOPT);
@@ -99,6 +105,14 @@ static __always_inline __pure bool use_fxsr(void)
         return static_cpu_has(X86_FEATURE_FXSR);
 }
 
+static inline void fx_finit(struct i387_fxsave_struct *fx)
+{
+       memset(fx, 0, xstate_size);
+       fx->cwd = 0x37f;
+       if (cpu_has_xmm)
+               fx->mxcsr = MXCSR_DEFAULT;
+}
+
 extern void __sanitize_i387_state(struct task_struct *);
 
 static inline void sanitize_i387_state(struct task_struct *tsk)
@@ -291,13 +305,13 @@ static inline void __thread_set_has_fpu(struct task_struct *tsk)
 static inline void __thread_fpu_end(struct task_struct *tsk)
 {
        __thread_clear_has_fpu(tsk);
-       if (!use_xsave())
+       if (!use_eager_fpu())
                stts();
 }
 
 static inline void __thread_fpu_begin(struct task_struct *tsk)
 {
-       if (!use_xsave())
+       if (!use_eager_fpu())
                clts();
        __thread_set_has_fpu(tsk);
 }
@@ -327,10 +341,14 @@ static inline void drop_fpu(struct task_struct *tsk)
 
 static inline void drop_init_fpu(struct task_struct *tsk)
 {
-       if (!use_xsave())
+       if (!use_eager_fpu())
                drop_fpu(tsk);
-       else
-               xrstor_state(init_xstate_buf, -1);
+       else {
+               if (use_xsave())
+                       xrstor_state(init_xstate_buf, -1);
+               else
+                       fxrstor_checking(&init_xstate_buf->i387);
+       }
 }
 
 /*
@@ -370,7 +388,7 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta
         * If the task has used the math, pre-load the FPU on xsave processors
         * or if the past 5 consecutive context-switches used math.
         */
-       fpu.preload = tsk_used_math(new) && (use_xsave() ||
+       fpu.preload = tsk_used_math(new) && (use_eager_fpu() ||
                                             new->fpu_counter > 5);
        if (__thread_has_fpu(old)) {
                if (!__save_init_fpu(old))
@@ -383,14 +401,14 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta
                        new->fpu_counter++;
                        __thread_set_has_fpu(new);
                        prefetch(new->thread.fpu.state);
-               } else if (!use_xsave())
+               } else if (!use_eager_fpu())
                        stts();
        } else {
                old->fpu_counter = 0;
                old->thread.fpu.last_cpu = ~0;
                if (fpu.preload) {
                        new->fpu_counter++;
-                       if (!use_xsave() && fpu_lazy_restore(new, cpu))
+                       if (!use_eager_fpu() && fpu_lazy_restore(new, cpu))
                                fpu.preload = 0;
                        else
                                prefetch(new->thread.fpu.state);
@@ -452,6 +470,14 @@ static inline void user_fpu_begin(void)
        preempt_enable();
 }
 
+static inline void __save_fpu(struct task_struct *tsk)
+{
+       if (use_xsave())
+               xsave_state(&tsk->thread.fpu.state->xsave, -1);
+       else
+               fpu_fxsave(&tsk->thread.fpu);
+}
+
 /*
  * These disable preemption on their own and are safe
  */
@@ -459,8 +485,8 @@ static inline void save_init_fpu(struct task_struct *tsk)
 {
        WARN_ON_ONCE(!__thread_has_fpu(tsk));
 
-       if (use_xsave()) {
-               xsave_state(&tsk->thread.fpu.state->xsave, -1);
+       if (use_eager_fpu()) {
+               __save_fpu(tsk);
                return;
        }
 
@@ -526,11 +552,9 @@ static inline void fpu_free(struct fpu *fpu)
 
 static inline void fpu_copy(struct task_struct *dst, struct task_struct *src)
 {
-       if (use_xsave()) {
-               struct xsave_struct *xsave = &dst->thread.fpu.state->xsave;
-
-               memset(&xsave->xsave_hdr, 0, sizeof(struct xsave_hdr_struct));
-               xsave_state(xsave, -1);
+       if (use_eager_fpu()) {
+               memset(&dst->thread.fpu.state->xsave, 0, xstate_size);
+               __save_fpu(dst);
        } else {
                struct fpu *dfpu = &dst->thread.fpu;
                struct fpu *sfpu = &src->thread.fpu;
index a5fbc3c5fccc5e60d61d9ec8f1d3f9f60a2a0467..b0fe078614d88713a86e0dff75eb23870e6b7495 100644 (file)
@@ -1297,7 +1297,6 @@ void __cpuinit cpu_init(void)
        dbg_restore_debug_regs();
 
        fpu_init();
-       xsave_init();
 
        raw_local_save_flags(kernel_eflags);
 
@@ -1352,6 +1351,5 @@ void __cpuinit cpu_init(void)
        dbg_restore_debug_regs();
 
        fpu_init();
-       xsave_init();
 }
 #endif
index 528557470ddb4efcb8088c7ab29cc0295fb14cf6..6782e3983865c4daf1be142705e23072bdaae4d5 100644 (file)
@@ -22,9 +22,8 @@
 /*
  * Were we in an interrupt that interrupted kernel mode?
  *
- * For now, on xsave platforms we will return interrupted
- * kernel FPU as not-idle. TBD: As we use non-lazy FPU restore
- * for xsave platforms, ideally we can change the return value
+ * For now, with eagerfpu we will return interrupted kernel FPU
+ * state as not-idle. TBD: Ideally we can change the return value
  * to something like __thread_has_fpu(current). But we need to
  * be careful of doing __thread_clear_has_fpu() before saving
  * the FPU etc for supporting nested uses etc. For now, take
@@ -38,7 +37,7 @@
  */
 static inline bool interrupted_kernel_fpu_idle(void)
 {
-       if (use_xsave())
+       if (use_eager_fpu())
                return 0;
 
        return !__thread_has_fpu(current) &&
@@ -84,7 +83,7 @@ void kernel_fpu_begin(void)
                __save_init_fpu(me);
                __thread_clear_has_fpu(me);
                /* We do 'stts()' in kernel_fpu_end() */
-       } else if (!use_xsave()) {
+       } else if (!use_eager_fpu()) {
                this_cpu_write(fpu_owner_task, NULL);
                clts();
        }
@@ -93,7 +92,7 @@ EXPORT_SYMBOL(kernel_fpu_begin);
 
 void kernel_fpu_end(void)
 {
-       if (use_xsave())
+       if (use_eager_fpu())
                math_state_restore();
        else
                stts();
@@ -122,7 +121,6 @@ static void __cpuinit mxcsr_feature_mask_init(void)
 {
        unsigned long mask = 0;
 
-       clts();
        if (cpu_has_fxsr) {
                memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
                asm volatile("fxsave %0" : : "m" (fx_scratch));
@@ -131,7 +129,6 @@ static void __cpuinit mxcsr_feature_mask_init(void)
                        mask = 0x0000ffbf;
        }
        mxcsr_feature_mask &= mask;
-       stts();
 }
 
 static void __cpuinit init_thread_xstate(void)
@@ -185,9 +182,8 @@ void __cpuinit fpu_init(void)
                init_thread_xstate();
 
        mxcsr_feature_mask_init();
-       /* clean state in init */
-       current_thread_info()->status = 0;
-       clear_used_math();
+       xsave_init();
+       eager_fpu_init();
 }
 
 void fpu_finit(struct fpu *fpu)
@@ -198,12 +194,7 @@ void fpu_finit(struct fpu *fpu)
        }
 
        if (cpu_has_fxsr) {
-               struct i387_fxsave_struct *fx = &fpu->state->fxsave;
-
-               memset(fx, 0, xstate_size);
-               fx->cwd = 0x37f;
-               if (cpu_has_xmm)
-                       fx->mxcsr = MXCSR_DEFAULT;
+               fx_finit(&fpu->state->fxsave);
        } else {
                struct i387_fsave_struct *fp = &fpu->state->fsave;
                memset(fp, 0, xstate_size);
index c21e30f8923b9132597cdb9949ee95135284dd6f..dc3567e083f9f5ec3a9d524756d191b5125490cc 100644 (file)
@@ -156,7 +156,7 @@ void flush_thread(void)
         * Free the FPU state for non xsave platforms. They get reallocated
         * lazily at the first use.
         */
-       if (!use_xsave())
+       if (!use_eager_fpu())
                free_thread_xstate(tsk);
 }
 
index ac7d5275f6e8eaa9649a7776f9adbf1034a7fa4e..4f4aba0551b0833e0b0155205b4f576f94a6a488 100644 (file)
@@ -630,7 +630,7 @@ EXPORT_SYMBOL_GPL(math_state_restore);
 dotraplinkage void __kprobes
 do_device_not_available(struct pt_regs *regs, long error_code)
 {
-       BUG_ON(use_xsave());
+       BUG_ON(use_eager_fpu());
 
 #ifdef CONFIG_MATH_EMULATION
        if (read_cr0() & X86_CR0_EM) {
index e7752bd7cac8bc248203f26b151d66394a2cbc3e..c0afd2c43761629c4c8ccccee292d996de3de4e7 100644 (file)
@@ -400,7 +400,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
                        set_used_math();
                }
 
-               if (use_xsave())
+               if (use_eager_fpu())
                        math_state_restore();
 
                return err;
@@ -450,28 +450,10 @@ static void prepare_fx_sw_frame(void)
  */
 static inline void xstate_enable(void)
 {
-       clts();
        set_in_cr4(X86_CR4_OSXSAVE);
        xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
 }
 
-/*
- * This is same as math_state_restore(). But use_xsave() is not yet
- * patched to use math_state_restore().
- */
-static inline void init_restore_xstate(void)
-{
-       init_fpu(current);
-       __thread_fpu_begin(current);
-       xrstor_state(init_xstate_buf, -1);
-}
-
-static inline void xstate_enable_ap(void)
-{
-       xstate_enable();
-       init_restore_xstate();
-}
-
 /*
  * Record the offsets and sizes of different state managed by the xsave
  * memory layout.
@@ -500,17 +482,20 @@ static void __init setup_xstate_features(void)
 /*
  * setup the xstate image representing the init state
  */
-static void __init setup_xstate_init(void)
+static void __init setup_init_fpu_buf(void)
 {
-       setup_xstate_features();
-
        /*
         * Setup init_xstate_buf to represent the init state of
         * all the features managed by the xsave
         */
        init_xstate_buf = alloc_bootmem_align(xstate_size,
                                              __alignof__(struct xsave_struct));
-       init_xstate_buf->i387.mxcsr = MXCSR_DEFAULT;
+       fx_finit(&init_xstate_buf->i387);
+
+       if (!cpu_has_xsave)
+               return;
+
+       setup_xstate_features();
 
        /*
         * Init all the features state with header_bv being 0x0
@@ -523,6 +508,17 @@ static void __init setup_xstate_init(void)
        xsave_state(init_xstate_buf, -1);
 }
 
+static int disable_eagerfpu;
+static int __init eager_fpu_setup(char *s)
+{
+       if (!strcmp(s, "on"))
+               setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
+       else if (!strcmp(s, "off"))
+               disable_eagerfpu = 1;
+       return 1;
+}
+__setup("eagerfpu=", eager_fpu_setup);
+
 /*
  * Enable and initialize the xsave feature.
  */
@@ -559,15 +555,10 @@ static void __init xstate_enable_boot_cpu(void)
 
        update_regset_xstate_info(xstate_size, pcntxt_mask);
        prepare_fx_sw_frame();
-
-       setup_xstate_init();
+       setup_init_fpu_buf();
 
        pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x\n",
                pcntxt_mask, xstate_size);
-
-       current->thread.fpu.state =
-            alloc_bootmem_align(xstate_size, __alignof__(struct xsave_struct));
-       init_restore_xstate();
 }
 
 /*
@@ -586,6 +577,42 @@ void __cpuinit xsave_init(void)
                return;
 
        this_func = next_func;
-       next_func = xstate_enable_ap;
+       next_func = xstate_enable;
        this_func();
 }
+
+static inline void __init eager_fpu_init_bp(void)
+{
+       current->thread.fpu.state =
+           alloc_bootmem_align(xstate_size, __alignof__(struct xsave_struct));
+       if (!init_xstate_buf)
+               setup_init_fpu_buf();
+}
+
+void __cpuinit eager_fpu_init(void)
+{
+       static __refdata void (*boot_func)(void) = eager_fpu_init_bp;
+
+       clear_used_math();
+       current_thread_info()->status = 0;
+       if (!cpu_has_eager_fpu) {
+               stts();
+               return;
+       }
+
+       if (boot_func) {
+               boot_func();
+               boot_func = NULL;
+       }
+
+       /*
+        * This is same as math_state_restore(). But use_xsave() is
+        * not yet patched to use math_state_restore().
+        */
+       init_fpu(current);
+       __thread_fpu_begin(current);
+       if (cpu_has_xsave)
+               xrstor_state(init_xstate_buf, -1);
+       else
+               fxrstor_checking(&init_xstate_buf->i387);
+}