[PATCH] x86-64: Let oprofile reserve MSR on all CPUs
authorAndi Kleen <ak@suse.de>
Mon, 2 Apr 2007 10:14:12 +0000 (12:14 +0200)
committerAndi Kleen <andi@basil.nowhere.org>
Mon, 2 Apr 2007 10:14:12 +0000 (12:14 +0200)
The MSR reservation is per CPU and oprofile would only allocate them
on the CPU it was initialized on. Change this to handle all CPUs.

This also fixes a warning about unprotected use of smp_processor_id()
in preemptible kernels.

Signed-off-by: Andi Kleen <ak@suse.de>
arch/i386/kernel/nmi.c
arch/x86_64/kernel/nmi.c

index 14702427b10415fbec39f241802ceca233b2b055..851f3b1c5b1ce53ba1f5b02956e939f97e1decd2 100644 (file)
@@ -122,64 +122,129 @@ static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
 /* checks for a bit availability (hack for oprofile) */
 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
 {
+       int cpu;
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
-
-       return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
+       for_each_possible_cpu (cpu) {
+               if (test_bit(counter, &per_cpu(perfctr_nmi_owner, cpu)))
+                       return 0;
+       }
+       return 1;
 }
 
 /* checks the an msr for availability */
 int avail_to_resrv_perfctr_nmi(unsigned int msr)
 {
        unsigned int counter;
+       int cpu;
 
        counter = nmi_perfctr_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
+       for_each_possible_cpu (cpu) {
+               if (test_bit(counter, &per_cpu(perfctr_nmi_owner, cpu)))
+                       return 0;
+       }
+       return 1;
 }
 
-int reserve_perfctr_nmi(unsigned int msr)
+static int __reserve_perfctr_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_perfctr_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner)))
+       if (!test_and_set_bit(counter, &per_cpu(perfctr_nmi_owner, cpu)))
                return 1;
        return 0;
 }
 
-void release_perfctr_nmi(unsigned int msr)
+static void __release_perfctr_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_perfctr_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner));
+       clear_bit(counter, &per_cpu(perfctr_nmi_owner, cpu));
 }
 
-int reserve_evntsel_nmi(unsigned int msr)
+int reserve_perfctr_nmi(unsigned int msr)
+{
+       int cpu, i;
+       for_each_possible_cpu (cpu) {
+               if (!__reserve_perfctr_nmi(cpu, msr)) {
+                       for_each_possible_cpu (i) {
+                               if (i >= cpu)
+                                       break;
+                               __release_perfctr_nmi(i, msr);
+                       }
+                       return 0;
+               }
+       }
+       return 1;
+}
+
+void release_perfctr_nmi(unsigned int msr)
+{
+       int cpu;
+       for_each_possible_cpu (cpu) {
+               __release_perfctr_nmi(cpu, msr);
+       }
+}
+
+int __reserve_evntsel_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_evntsel_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner)[0]))
+       if (!test_and_set_bit(counter, &per_cpu(evntsel_nmi_owner, cpu)[0]))
                return 1;
        return 0;
 }
 
-void release_evntsel_nmi(unsigned int msr)
+static void __release_evntsel_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_evntsel_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner)[0]);
+       clear_bit(counter, &per_cpu(evntsel_nmi_owner, cpu)[0]);
+}
+
+int reserve_evntsel_nmi(unsigned int msr)
+{
+       int cpu, i;
+       for_each_possible_cpu (cpu) {
+               if (!__reserve_evntsel_nmi(cpu, msr)) {
+                       for_each_possible_cpu (i) {
+                               if (i >= cpu)
+                                       break;
+                               __release_evntsel_nmi(i, msr);
+                       }
+                       return 0;
+               }
+       }
+       return 1;
+}
+
+void release_evntsel_nmi(unsigned int msr)
+{
+       int cpu;
+       for_each_possible_cpu (cpu) {
+               __release_evntsel_nmi(cpu, msr);
+       }
 }
 
 static __cpuinit inline int nmi_known_cpu(void)
@@ -507,10 +572,10 @@ static int setup_k7_watchdog(void)
 
        perfctr_msr = MSR_K7_PERFCTR0;
        evntsel_msr = MSR_K7_EVNTSEL0;
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        wrmsrl(perfctr_msr, 0UL);
@@ -533,7 +598,7 @@ static int setup_k7_watchdog(void)
        wd->check_bit = 1ULL<<63;
        return 1;
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -544,8 +609,8 @@ static void stop_k7_watchdog(void)
 
        wrmsr(wd->evntsel_msr, 0, 0);
 
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 #define P6_EVNTSEL0_ENABLE     (1 << 22)
@@ -563,10 +628,10 @@ static int setup_p6_watchdog(void)
 
        perfctr_msr = MSR_P6_PERFCTR0;
        evntsel_msr = MSR_P6_EVNTSEL0;
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        wrmsrl(perfctr_msr, 0UL);
@@ -590,7 +655,7 @@ static int setup_p6_watchdog(void)
        wd->check_bit = 1ULL<<39;
        return 1;
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -601,8 +666,8 @@ static void stop_p6_watchdog(void)
 
        wrmsr(wd->evntsel_msr, 0, 0);
 
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 /* Note that these events don't tick when the CPU idles. This means
@@ -668,10 +733,10 @@ static int setup_p4_watchdog(void)
                cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4);
        }
 
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        evntsel = P4_ESCR_EVENT_SELECT(0x3F)
@@ -695,7 +760,7 @@ static int setup_p4_watchdog(void)
        wd->check_bit = 1ULL<<39;
        return 1;
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -707,8 +772,8 @@ static void stop_p4_watchdog(void)
        wrmsr(wd->cccr_msr, 0, 0);
        wrmsr(wd->evntsel_msr, 0, 0);
 
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 #define ARCH_PERFMON_NMI_EVENT_SEL     ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
@@ -736,10 +801,10 @@ static int setup_intel_arch_watchdog(void)
        perfctr_msr = MSR_ARCH_PERFMON_PERFCTR0;
        evntsel_msr = MSR_ARCH_PERFMON_EVENTSEL0;
 
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        wrmsrl(perfctr_msr, 0UL);
@@ -764,7 +829,7 @@ static int setup_intel_arch_watchdog(void)
        wd->check_bit = 1ULL << (eax.split.bit_width - 1);
        return 1;
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -787,8 +852,8 @@ static void stop_intel_arch_watchdog(void)
                return;
 
        wrmsr(wd->evntsel_msr, 0, 0);
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 void setup_apic_nmi_watchdog (void *unused)
index 82d9d85d52709c2322392a5e33e5768b0c8dc085..17cf2d6b5e99c4a053064127b5087dd104354ed4 100644 (file)
@@ -108,64 +108,128 @@ static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
 /* checks for a bit availability (hack for oprofile) */
 int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
 {
+       int cpu;
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
-
-       return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
+       for_each_possible_cpu (cpu) {
+               if (test_bit(counter, &per_cpu(perfctr_nmi_owner, cpu)))
+                       return 0;
+       }
+       return 1;
 }
 
 /* checks the an msr for availability */
 int avail_to_resrv_perfctr_nmi(unsigned int msr)
 {
        unsigned int counter;
+       int cpu;
 
        counter = nmi_perfctr_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       return (!test_bit(counter, &__get_cpu_var(perfctr_nmi_owner)));
+       for_each_possible_cpu (cpu) {
+               if (test_bit(counter, &per_cpu(perfctr_nmi_owner, cpu)))
+                       return 0;
+       }
+       return 1;
 }
 
-int reserve_perfctr_nmi(unsigned int msr)
+static int __reserve_perfctr_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_perfctr_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       if (!test_and_set_bit(counter, &__get_cpu_var(perfctr_nmi_owner)))
+       if (!test_and_set_bit(counter, &per_cpu(perfctr_nmi_owner, cpu)))
                return 1;
        return 0;
 }
 
-void release_perfctr_nmi(unsigned int msr)
+static void __release_perfctr_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_perfctr_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       clear_bit(counter, &__get_cpu_var(perfctr_nmi_owner));
+       clear_bit(counter, &per_cpu(perfctr_nmi_owner, cpu));
 }
 
-int reserve_evntsel_nmi(unsigned int msr)
+int reserve_perfctr_nmi(unsigned int msr)
+{
+       int cpu, i;
+       for_each_possible_cpu (cpu) {
+               if (!__reserve_perfctr_nmi(cpu, msr)) {
+                       for_each_possible_cpu (i) {
+                               if (i >= cpu)
+                                       break;
+                               __release_perfctr_nmi(i, msr);
+                       }
+                       return 0;
+               }
+       }
+       return 1;
+}
+
+void release_perfctr_nmi(unsigned int msr)
+{
+       int cpu;
+       for_each_possible_cpu (cpu)
+               __release_perfctr_nmi(cpu, msr);
+}
+
+int __reserve_evntsel_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_evntsel_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       if (!test_and_set_bit(counter, &__get_cpu_var(evntsel_nmi_owner)))
+       if (!test_and_set_bit(counter, &per_cpu(evntsel_nmi_owner, cpu)[0]))
                return 1;
        return 0;
 }
 
-void release_evntsel_nmi(unsigned int msr)
+static void __release_evntsel_nmi(int cpu, unsigned int msr)
 {
        unsigned int counter;
+       if (cpu < 0)
+               cpu = smp_processor_id();
 
        counter = nmi_evntsel_msr_to_bit(msr);
        BUG_ON(counter > NMI_MAX_COUNTER_BITS);
 
-       clear_bit(counter, &__get_cpu_var(evntsel_nmi_owner));
+       clear_bit(counter, &per_cpu(evntsel_nmi_owner, cpu)[0]);
+}
+
+int reserve_evntsel_nmi(unsigned int msr)
+{
+       int cpu, i;
+       for_each_possible_cpu (cpu) {
+               if (!__reserve_evntsel_nmi(cpu, msr)) {
+                       for_each_possible_cpu (i) {
+                               if (i >= cpu)
+                                       break;
+                               __release_evntsel_nmi(i, msr);
+                       }
+                       return 0;
+               }
+       }
+       return 1;
+}
+
+void release_evntsel_nmi(unsigned int msr)
+{
+       int cpu;
+       for_each_possible_cpu (cpu) {
+               __release_evntsel_nmi(cpu, msr);
+       }
 }
 
 static __cpuinit inline int nmi_known_cpu(void)
@@ -472,10 +536,10 @@ static int setup_k7_watchdog(void)
 
        perfctr_msr = MSR_K7_PERFCTR0;
        evntsel_msr = MSR_K7_EVNTSEL0;
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        /* Simulator may not support it */
@@ -501,9 +565,9 @@ static int setup_k7_watchdog(void)
        wd->check_bit = 1ULL<<63;
        return 1;
 fail2:
-       release_evntsel_nmi(evntsel_msr);
+       __release_evntsel_nmi(-1, evntsel_msr);
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -514,8 +578,8 @@ static void stop_k7_watchdog(void)
 
        wrmsr(wd->evntsel_msr, 0, 0);
 
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 /* Note that these events don't tick when the CPU idles. This means
@@ -581,10 +645,10 @@ static int setup_p4_watchdog(void)
                cccr_val = P4_CCCR_OVF_PMI1 | P4_CCCR_ESCR_SELECT(4);
        }
 
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        evntsel = P4_ESCR_EVENT_SELECT(0x3F)
@@ -609,7 +673,7 @@ static int setup_p4_watchdog(void)
        wd->check_bit = 1ULL<<39;
        return 1;
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -621,8 +685,8 @@ static void stop_p4_watchdog(void)
        wrmsr(wd->cccr_msr, 0, 0);
        wrmsr(wd->evntsel_msr, 0, 0);
 
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 #define ARCH_PERFMON_NMI_EVENT_SEL     ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL
@@ -650,10 +714,10 @@ static int setup_intel_arch_watchdog(void)
        perfctr_msr = MSR_ARCH_PERFMON_PERFCTR0;
        evntsel_msr = MSR_ARCH_PERFMON_EVENTSEL0;
 
-       if (!reserve_perfctr_nmi(perfctr_msr))
+       if (!__reserve_perfctr_nmi(-1, perfctr_msr))
                goto fail;
 
-       if (!reserve_evntsel_nmi(evntsel_msr))
+       if (!__reserve_evntsel_nmi(-1, evntsel_msr))
                goto fail1;
 
        wrmsrl(perfctr_msr, 0UL);
@@ -680,7 +744,7 @@ static int setup_intel_arch_watchdog(void)
        wd->check_bit = 1ULL << (eax.split.bit_width - 1);
        return 1;
 fail1:
-       release_perfctr_nmi(perfctr_msr);
+       __release_perfctr_nmi(-1, perfctr_msr);
 fail:
        return 0;
 }
@@ -704,8 +768,8 @@ static void stop_intel_arch_watchdog(void)
 
        wrmsr(wd->evntsel_msr, 0, 0);
 
-       release_evntsel_nmi(wd->evntsel_msr);
-       release_perfctr_nmi(wd->perfctr_msr);
+       __release_evntsel_nmi(-1, wd->evntsel_msr);
+       __release_perfctr_nmi(-1, wd->perfctr_msr);
 }
 
 void setup_apic_nmi_watchdog(void *unused)