x86: Clean up cr4 manipulation

CR4 manipulation was split, seemingly at random, between direct
(write_cr4) and using a helper (set/clear_in_cr4).  Unfortunately,
the set_in_cr4 and clear_in_cr4 helpers also poke at the boot code,
which only a small subset of users actually wanted.

This patch replaces all cr4 access in functions that don't leave cr4
exactly the way they found it with new helpers cr4_set_bits,
cr4_clear_bits, and cr4_set_bits_and_update_boot.

Signed-off-by: Andy Lutomirski <luto@amacapital.net>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Vince Weaver <vince@deater.net>
Cc: "hillf.zj" <hillf.zj@alibaba-inc.com>
Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Arnaldo Carvalho de Melo <acme@kernel.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/495a10bdc9e67016b8fd3945700d46cfd5c12c2f.1414190806.git.luto@amacapital.net
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index a092a0cce0b759a20ce906efc03e3cb5e8d1d2af..ec1c93588cefd0c4e3a4705c6d966e2555ba95aa 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -579,39 +579,6 @@ static inline void load_sp0(struct tss_struct *tss,
 #define set_iopl_mask native_set_iopl_mask
 #endif /* CONFIG_PARAVIRT */
 
-/*
- * Save the cr4 feature set we're using (ie
- * Pentium 4MB enable and PPro Global page
- * enable), so that any CPU's that boot up
- * after us can get the correct flags.
- */
-extern unsigned long mmu_cr4_features;
-extern u32 *trampoline_cr4_features;
-
-static inline void set_in_cr4(unsigned long mask)
-{
-       unsigned long cr4;
-
-       mmu_cr4_features |= mask;
-       if (trampoline_cr4_features)
-               *trampoline_cr4_features = mmu_cr4_features;
-       cr4 = read_cr4();
-       cr4 |= mask;
-       write_cr4(cr4);
-}
-
-static inline void clear_in_cr4(unsigned long mask)
-{
-       unsigned long cr4;
-
-       mmu_cr4_features &= ~mask;
-       if (trampoline_cr4_features)
-               *trampoline_cr4_features = mmu_cr4_features;
-       cr4 = read_cr4();
-       cr4 &= ~mask;
-       write_cr4(cr4);
-}
-
 typedef struct {
        unsigned long           seg;
 } mm_segment_t;

diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 04905bfc508b9925c697687b7c1d5754827ba417..fc0c4bc356ce45ad663eae11a2d433a118ef38c3 100644 (file)
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -15,6 +15,43 @@
 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
 #endif
 
+/* Set in this cpu's CR4. */
+static inline void cr4_set_bits(unsigned long mask)
+{
+       unsigned long cr4;
+
+       cr4 = read_cr4();
+       cr4 |= mask;
+       write_cr4(cr4);
+}
+
+/* Clear in this cpu's CR4. */
+static inline void cr4_clear_bits(unsigned long mask)
+{
+       unsigned long cr4;
+
+       cr4 = read_cr4();
+       cr4 &= ~mask;
+       write_cr4(cr4);
+}
+
+/*
+ * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
+ * enable and PPro Global page enable), so that any CPU's that boot
+ * up after us can get the correct flags.  This should only be used
+ * during boot on the boot cpu.
+ */
+extern unsigned long mmu_cr4_features;
+extern u32 *trampoline_cr4_features;
+
+static inline void cr4_set_bits_and_update_boot(unsigned long mask)
+{
+       mmu_cr4_features |= mask;
+       if (trampoline_cr4_features)
+               *trampoline_cr4_features = mmu_cr4_features;
+       cr4_set_bits(mask);
+}
+
 static inline void __native_flush_tlb(void)
 {
        native_write_cr3(native_read_cr3());

diff --git a/arch/x86/include/asm/virtext.h b/arch/x86/include/asm/virtext.h
index 5da71c27cc59e47485bcdee1bcce343d1118adf9..f41e19ca717b0bbffb5d9f0f7e3402113a77f76e 100644 (file)
--- a/arch/x86/include/asm/virtext.h
+++ b/arch/x86/include/asm/virtext.h
@@ -19,6 +19,7 @@
 
 #include <asm/vmx.h>
 #include <asm/svm.h>
+#include <asm/tlbflush.h>
 
 /*
  * VMX functions:
@@ -40,7 +41,7 @@ static inline int cpu_has_vmx(void)
 static inline void cpu_vmxoff(void)
 {
        asm volatile (ASM_VMX_VMXOFF : : : "cc");
-       write_cr4(read_cr4() & ~X86_CR4_VMXE);
+       cr4_clear_bits(X86_CR4_VMXE);
 }
 
 static inline int cpu_vmx_enabled(void)

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c6049650c093f79f849c4779d7800601311eb4d3..9d8fc49f092277719363234939ea737e1a6a0b50 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -278,7 +278,7 @@ __setup("nosmep", setup_disable_smep);
 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
 {
        if (cpu_has(c, X86_FEATURE_SMEP))
-               set_in_cr4(X86_CR4_SMEP);
+               cr4_set_bits(X86_CR4_SMEP);
 }
 
 static __init int setup_disable_smap(char *arg)
@@ -298,9 +298,9 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c)
 
        if (cpu_has(c, X86_FEATURE_SMAP)) {
 #ifdef CONFIG_X86_SMAP
-               set_in_cr4(X86_CR4_SMAP);
+               cr4_set_bits(X86_CR4_SMAP);
 #else
-               clear_in_cr4(X86_CR4_SMAP);
+               cr4_clear_bits(X86_CR4_SMAP);
 #endif
        }
 }
@@ -1312,7 +1312,7 @@ void cpu_init(void)
 
        pr_debug("Initializing CPU#%d\n", cpu);
 
-       clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
+       cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
 
        /*
         * Initialize the per-CPU GDT with the boot GDT,
@@ -1393,7 +1393,7 @@ void cpu_init(void)
        printk(KERN_INFO "Initializing CPU#%d\n", cpu);
 
        if (cpu_feature_enabled(X86_FEATURE_VME) || cpu_has_tsc || cpu_has_de)
-               clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
+               cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
 
        load_current_idt();
        switch_to_new_gdt(cpu);

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index d23179900755a33e2c337f5211b8a867053846d3..15ad3ed1a3cd3f4e7f626c2f77c079f6d0de3249 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -44,6 +44,7 @@
 
 #include <asm/processor.h>
 #include <asm/traps.h>
+#include <asm/tlbflush.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 
@@ -1449,7 +1450,7 @@ static void __mcheck_cpu_init_generic(void)
        bitmap_fill(all_banks, MAX_NR_BANKS);
        machine_check_poll(MCP_UC | m_fl, &all_banks);
 
-       set_in_cr4(X86_CR4_MCE);
+       cr4_set_bits(X86_CR4_MCE);
 
        rdmsrl(MSR_IA32_MCG_CAP, cap);
        if (cap & MCG_CTL_P)

diff --git a/arch/x86/kernel/cpu/mcheck/p5.c b/arch/x86/kernel/cpu/mcheck/p5.c
index ec2663a708e40d2e3fa03b36fa7587c860a1871b..737b0ad4e61ae2f41312039aa012d8b89db2f671 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/p5.c
+++ b/arch/x86/kernel/cpu/mcheck/p5.c
@@ -9,6 +9,7 @@
 
 #include <asm/processor.h>
 #include <asm/traps.h>
+#include <asm/tlbflush.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 
@@ -65,7 +66,7 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
               "Intel old style machine check architecture supported.\n");
 
        /* Enable MCE: */
-       set_in_cr4(X86_CR4_MCE);
+       cr4_set_bits(X86_CR4_MCE);
        printk(KERN_INFO
               "Intel old style machine check reporting enabled on CPU#%d.\n",
               smp_processor_id());

diff --git a/arch/x86/kernel/cpu/mcheck/winchip.c b/arch/x86/kernel/cpu/mcheck/winchip.c
index bd5d46a32210a15deb8895c37e67e5e0dea9d7c5..44f138296fbebdf2a2e6ae25b5b9d94f112102ef 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/winchip.c
+++ b/arch/x86/kernel/cpu/mcheck/winchip.c
@@ -8,6 +8,7 @@
 
 #include <asm/processor.h>
 #include <asm/traps.h>
+#include <asm/tlbflush.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 
@@ -36,7 +37,7 @@ void winchip_mcheck_init(struct cpuinfo_x86 *c)
        lo &= ~(1<<4);  /* Enable MCE */
        wrmsr(MSR_IDT_FCR1, lo, hi);
 
-       set_in_cr4(X86_CR4_MCE);
+       cr4_set_bits(X86_CR4_MCE);
 
        printk(KERN_INFO
               "Winchip machine check reporting enabled on CPU#0.\n");

diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 143e5f5dc8551b568b7eabb84d83da764268d6b2..6b5acd5f4a34a187ed3da83fa4281061123b8945 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -31,6 +31,7 @@
 #include <asm/nmi.h>
 #include <asm/smp.h>
 #include <asm/alternative.h>
+#include <asm/tlbflush.h>
 #include <asm/timer.h>
 #include <asm/desc.h>
 #include <asm/ldt.h>
@@ -1328,7 +1329,7 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
 
        case CPU_STARTING:
                if (x86_pmu.attr_rdpmc)
-                       set_in_cr4(X86_CR4_PCE);
+                       cr4_set_bits(X86_CR4_PCE);
                if (x86_pmu.cpu_starting)
                        x86_pmu.cpu_starting(cpu);
                break;
@@ -1834,9 +1835,9 @@ static void change_rdpmc(void *info)
        bool enable = !!(unsigned long)info;
 
        if (enable)
-               set_in_cr4(X86_CR4_PCE);
+               cr4_set_bits(X86_CR4_PCE);
        else
-               clear_in_cr4(X86_CR4_PCE);
+               cr4_clear_bits(X86_CR4_PCE);
 }
 
 static ssize_t set_attr_rdpmc(struct device *cdev,

diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index a9a4229f6161b25b6e8a5752be7943f478525d21..87727b03196da51aa6aaf68de607485d262e5fab 100644 (file)
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -13,6 +13,7 @@
 #include <asm/sigcontext.h>
 #include <asm/processor.h>
 #include <asm/math_emu.h>
+#include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 #include <asm/ptrace.h>
 #include <asm/i387.h>
@@ -180,7 +181,7 @@ void fpu_init(void)
        if (cpu_has_xmm)
                cr4_mask |= X86_CR4_OSXMMEXCPT;
        if (cr4_mask)
-               set_in_cr4(cr4_mask);
+               cr4_set_bits(cr4_mask);
 
        cr0 = read_cr0();
        cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index e127ddaa2d5ad3e139ff7a99fb5485b43fb19416..046e2d620bbe7be507808e0f7188c45249f2d69c 100644 (file)
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -28,6 +28,7 @@
 #include <asm/fpu-internal.h>
 #include <asm/debugreg.h>
 #include <asm/nmi.h>
+#include <asm/tlbflush.h>
 
 /*
  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
@@ -141,7 +142,7 @@ void flush_thread(void)
 
 static void hard_disable_TSC(void)
 {
-       write_cr4(read_cr4() | X86_CR4_TSD);
+       cr4_set_bits(X86_CR4_TSD);
 }
 
 void disable_TSC(void)
@@ -158,7 +159,7 @@ void disable_TSC(void)
 
 static void hard_enable_TSC(void)
 {
-       write_cr4(read_cr4() & ~X86_CR4_TSD);
+       cr4_clear_bits(X86_CR4_TSD);
 }
 
 static void enable_TSC(void)

diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 0de1fae2bdf000b5ed6cfae68d76ca21d5c246b9..34f66e58a896693d392e4ed074b1f9ca664974d3 100644 (file)
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -12,6 +12,7 @@
 #include <asm/i387.h>
 #include <asm/fpu-internal.h>
 #include <asm/sigframe.h>
+#include <asm/tlbflush.h>
 #include <asm/xcr.h>
 
 /*
@@ -453,7 +454,7 @@ static void prepare_fx_sw_frame(void)
  */
 static inline void xstate_enable(void)
 {
-       set_in_cr4(X86_CR4_OSXSAVE);
+       cr4_set_bits(X86_CR4_OSXSAVE);
        xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
 }
 

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index d4c58d884838d1539cb52e13df3c6893459a98af..db77537013d1c246345a1c381e961b4abbed7b26 100644 (file)
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2812,7 +2812,7 @@ static int hardware_enable(void)
                /* enable and lock */
                wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
        }
-       write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
+       cr4_set_bits(X86_CR4_VMXE);
 
        if (vmm_exclusive) {
                kvm_cpu_vmxon(phys_addr);
@@ -2849,7 +2849,7 @@ static void hardware_disable(void)
                vmclear_local_loaded_vmcss();
                kvm_cpu_vmxoff();
        }
-       write_cr4(read_cr4() & ~X86_CR4_VMXE);
+       cr4_clear_bits(X86_CR4_VMXE);
 }
 
 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,

diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index 079c3b6a3ff181277a7cb4270895f27d9a1d6f8b..d4eddbd92c287c73393e3dea524c4baa2a578f5b 100644 (file)
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -173,11 +173,11 @@ static void __init probe_page_size_mask(void)
 
        /* Enable PSE if available */
        if (cpu_has_pse)
-               set_in_cr4(X86_CR4_PSE);
+               cr4_set_bits_and_update_boot(X86_CR4_PSE);
 
        /* Enable PGE if available */
        if (cpu_has_pge) {
-               set_in_cr4(X86_CR4_PGE);
+               cr4_set_bits_and_update_boot(X86_CR4_PGE);
                __supported_pte_mask |= _PAGE_GLOBAL;
        }
 }

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 78a881b7fc415e16f50f16e4381e968370f4fd9f..bd8b8459c3d05923286b4dc6b5ebb7e37e51d926 100644 (file)
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1494,10 +1494,10 @@ static void xen_pvh_set_cr_flags(int cpu)
         * set them here. For all, OSFXSR OSXMMEXCPT are set in fpu_init.
        */
        if (cpu_has_pse)
-               set_in_cr4(X86_CR4_PSE);
+               cr4_set_bits_and_update_boot(X86_CR4_PSE);
 
        if (cpu_has_pge)
-               set_in_cr4(X86_CR4_PGE);
+               cr4_set_bits_and_update_boot(X86_CR4_PGE);
 }
 
 /*

diff --git a/drivers/lguest/x86/core.c b/drivers/lguest/x86/core.c
index 922a1acbf652b9376e4b2dd39ba6be2859307594..6adfd7ba4c977434221d859185772f26530c8bc7 100644 (file)
--- a/drivers/lguest/x86/core.c
+++ b/drivers/lguest/x86/core.c
@@ -47,6 +47,7 @@
 #include <asm/lguest.h>
 #include <asm/uaccess.h>
 #include <asm/i387.h>
+#include <asm/tlbflush.h>
 #include "../lg.h"
 
 static int cpu_had_pge;
@@ -452,9 +453,9 @@ void lguest_arch_handle_trap(struct lg_cpu *cpu)
 static void adjust_pge(void *on)
 {
        if (on)
-               write_cr4(read_cr4() | X86_CR4_PGE);
+               cr4_set_bits(X86_CR4_PGE);
        else
-               write_cr4(read_cr4() & ~X86_CR4_PGE);
+               cr4_clear_bits(X86_CR4_PGE);
 }
 
 /*H:020