x86, pat/mtrr: Rendezvous all the cpus for MTRR/PAT init

SDM Vol 3a section titled "MTRR considerations in MP systems" specifies
the need for synchronizing the logical cpu's while initializing/updating
MTRR.

Currently Linux kernel does the synchronization of all cpu's only when
a single MTRR register is programmed/updated. During an AP online
(during boot/cpu-online/resume)  where we initialize all the MTRR/PAT registers,
we don't follow this synchronization algorithm.

This can lead to scenarios where during a dynamic cpu online, that logical cpu
is initializing MTRR/PAT with cache disabled (cr0.cd=1) etc while other logical
HT sibling continue to run (also with cache disabled because of cr0.cd=1
on its sibling).

Starting from Westmere, VMX transitions with cr0.cd=1 don't work properly
(because of some VMX performance optimizations) and the above scenario
(with one logical cpu doing VMX activity and another logical cpu coming online)
can result in system crash.

Fix the MTRR initialization by doing rendezvous of all the cpus. During
boot and resume, we delay the MTRR/PAT init for APs till all the
logical cpu's come online and the rendezvous process at the end of AP's bringup,
will initialize the MTRR/PAT for all AP's.

For dynamic single cpu online, we synchronize all the logical cpus and
do the MTRR/PAT init on the AP that is coming online.

Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

diff --git a/arch/x86/include/asm/mtrr.h b/arch/x86/include/asm/mtrr.h
index a51ada8467de36ddcf729b7f099455225842d28a..d5366ec5cb8f17c07d91987a59abbacfdb89a5a8 100644 (file)
--- a/arch/x86/include/asm/mtrr.h
+++ b/arch/x86/include/asm/mtrr.h
@@ -121,8 +121,12 @@ extern int mtrr_del_page(int reg, unsigned long base, unsigned long size);
 extern void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi);
 extern void mtrr_ap_init(void);
 extern void mtrr_bp_init(void);
+extern void set_mtrr_aps_delayed_init(void);
+extern void mtrr_aps_init(void);
+extern void mtrr_bp_restore(void);
 extern int mtrr_trim_uncached_memory(unsigned long end_pfn);
 extern int amd_special_default_mtrr(void);
+extern u32 mtrr_aps_delayed_init;
 #  else
 static inline u8 mtrr_type_lookup(u64 addr, u64 end)
 {
@@ -161,6 +165,9 @@ static inline void mtrr_centaur_report_mcr(int mcr, u32 lo, u32 hi)
 
 #define mtrr_ap_init() do {} while (0)
 #define mtrr_bp_init() do {} while (0)
+#define set_mtrr_aps_delayed_init() do {} while (0)
+#define mtrr_aps_init() do {} while (0)
+#define mtrr_bp_restore() do {} while (0)
 #  endif
 
 #ifdef CONFIG_COMPAT

diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 7af0f88a41630bb67bb78ecf476870fa70b5ed1d..7339be0aa58003543c515809c80d138343a612c7 100644 (file)
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -58,6 +58,7 @@ unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
 static DEFINE_MUTEX(mtrr_mutex);
 
 u64 size_or_mask, size_and_mask;
+u32 mtrr_aps_delayed_init;
 
 static struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
 
@@ -163,7 +164,10 @@ static void ipi_handler(void *info)
        if (data->smp_reg != ~0U) {
                mtrr_if->set(data->smp_reg, data->smp_base,
                             data->smp_size, data->smp_type);
-       } else {
+       } else if (mtrr_aps_delayed_init) {
+               /*
+                * Initialize the MTRRs inaddition to the synchronisation.
+                */
                mtrr_if->set_all();
        }
 
@@ -265,6 +269,8 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ
         */
        if (reg != ~0U)
                mtrr_if->set(reg, base, size, type);
+       else if (!mtrr_aps_delayed_init)
+               mtrr_if->set_all();
 
        /* Wait for the others */
        while (atomic_read(&data.count))
@@ -721,9 +727,7 @@ void __init mtrr_bp_init(void)
 
 void mtrr_ap_init(void)
 {
-       unsigned long flags;
-
-       if (!mtrr_if || !use_intel())
+       if (!use_intel() || mtrr_aps_delayed_init)
                return;
        /*
         * Ideally we should hold mtrr_mutex here to avoid mtrr entries
@@ -738,11 +742,7 @@ void mtrr_ap_init(void)
         *   2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
         *      lock to prevent mtrr entry changes
         */
-       local_irq_save(flags);
-
-       mtrr_if->set_all();
-
-       local_irq_restore(flags);
+       set_mtrr(~0U, 0, 0, 0);
 }
 
 /**
@@ -753,6 +753,34 @@ void mtrr_save_state(void)
        smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
 }
 
+void set_mtrr_aps_delayed_init(void)
+{
+       if (!use_intel())
+               return;
+
+       mtrr_aps_delayed_init = 1;
+}
+
+/*
+ * MTRR initialization for all AP's
+ */
+void mtrr_aps_init(void)
+{
+       if (!use_intel())
+               return;
+
+       set_mtrr(~0U, 0, 0, 0);
+       mtrr_aps_delayed_init = 0;
+}
+
+void mtrr_bp_restore(void)
+{
+       if (!use_intel())
+               return;
+
+       mtrr_if->set_all();
+}
+
 static int __init mtrr_init_finialize(void)
 {
        if (!mtrr_if)

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 2fecda69ee646a78402cc6a1d9e3d86a29ca06d6..d720b7e0cf3db38c13b5a720239eceaf7be3535d 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1116,9 +1116,22 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 
        if (is_uv_system())
                uv_system_init();
+
+       set_mtrr_aps_delayed_init();
 out:
        preempt_enable();
 }
+
+void arch_enable_nonboot_cpus_begin(void)
+{
+       set_mtrr_aps_delayed_init();
+}
+
+void arch_enable_nonboot_cpus_end(void)
+{
+       mtrr_aps_init();
+}
+
 /*
  * Early setup to make printk work.
  */
@@ -1140,6 +1153,7 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
        setup_ioapic_dest();
 #endif
        check_nmi_watchdog();
+       mtrr_aps_init();
 }
 
 static int __initdata setup_possible_cpus = -1;

diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index b3d20b9cac6366922d189572396d7c805c6680b2..417c9f5b4afa700477ec29d26ac41945828eddfb 100644 (file)
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -242,7 +242,7 @@ static void __restore_processor_state(struct saved_context *ctxt)
        fix_processor_context();
 
        do_fpu_end();
-       mtrr_ap_init();
+       mtrr_bp_restore();
 
 #ifdef CONFIG_X86_OLD_MCE
        mcheck_init(&boot_cpu_data);

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 8ce10043e4aca1d9d48ab9f127e6079c8634f9fd..f5f9485b8c0f9a30f4084b0c9a8846f74e05d72d 100644 (file)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -413,6 +413,14 @@ int disable_nonboot_cpus(void)
        return error;
 }
 
+void __weak arch_enable_nonboot_cpus_begin(void)
+{
+}
+
+void __weak arch_enable_nonboot_cpus_end(void)
+{
+}
+
 void __ref enable_nonboot_cpus(void)
 {
        int cpu, error;
@@ -424,6 +432,9 @@ void __ref enable_nonboot_cpus(void)
                goto out;
 
        printk("Enabling non-boot CPUs ...\n");
+
+       arch_enable_nonboot_cpus_begin();
+
        for_each_cpu(cpu, frozen_cpus) {
                error = _cpu_up(cpu, 1);
                if (!error) {
@@ -432,6 +443,9 @@ void __ref enable_nonboot_cpus(void)
                }
                printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
        }
+
+       arch_enable_nonboot_cpus_end();
+
        cpumask_clear(frozen_cpus);
 out:
        cpu_maps_update_done();