Check whether the TLB operations need broadcasting on SMP systems

ARMv7 SMP hardware can handle the TLB maintenance operations
broadcasting in hardware so that the software can avoid the costly IPIs.
This patch adds the necessary checks (the MMFR3 CPUID register) to avoid
the broadcasting if already supported by the hardware.

(this patch is based on the work done by Tony Thompson @ ARM)

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h
index 7b9d27e749b8e8afefbff6a292bbb334adcac636..b3e656c6fb78e20fb58fb2d361e4a2b968bfe8b7 100644 (file)
--- a/arch/arm/include/asm/cputype.h
+++ b/arch/arm/include/asm/cputype.h
@@ -8,6 +8,21 @@
 #define CPUID_TCM      2
 #define CPUID_TLBTYPE  3
 
+#define CPUID_EXT_PFR0 "c1, 0"
+#define CPUID_EXT_PFR1 "c1, 1"
+#define CPUID_EXT_DFR0 "c1, 2"
+#define CPUID_EXT_AFR0 "c1, 3"
+#define CPUID_EXT_MMFR0        "c1, 4"
+#define CPUID_EXT_MMFR1        "c1, 5"
+#define CPUID_EXT_MMFR2        "c1, 6"
+#define CPUID_EXT_MMFR3        "c1, 7"
+#define CPUID_EXT_ISAR0        "c2, 0"
+#define CPUID_EXT_ISAR1        "c2, 1"
+#define CPUID_EXT_ISAR2        "c2, 2"
+#define CPUID_EXT_ISAR3        "c2, 3"
+#define CPUID_EXT_ISAR4        "c2, 4"
+#define CPUID_EXT_ISAR5        "c2, 5"
+
 #ifdef CONFIG_CPU_CP15
 #define read_cpuid(reg)                                                        \
        ({                                                              \
@@ -18,9 +33,19 @@
                    : "cc");                                            \
                __val;                                                  \
        })
+#define read_cpuid_ext(ext_reg)                                                \
+       ({                                                              \
+               unsigned int __val;                                     \
+               asm("mrc        p15, 0, %0, c0, " ext_reg               \
+                   : "=r" (__val)                                      \
+                   :                                                   \
+                   : "cc");                                            \
+               __val;                                                  \
+       })
 #else
 extern unsigned int processor_id;
 #define read_cpuid(reg) (processor_id)
+#define read_cpuid_ext(reg) 0
 #endif
 
 /*

diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index a62218013c78bd80447013c5f9f8872c010cd638..c964f3fc3bc57e906c452cbbe059fed506c19ee8 100644 (file)
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -40,6 +40,12 @@
 #define TLB_V6_I_ASID  (1 << 18)
 
 #define TLB_BTB                (1 << 28)
+
+/* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
+#define TLB_V7_UIS_PAGE        (1 << 19)
+#define TLB_V7_UIS_FULL (1 << 20)
+#define TLB_V7_UIS_ASID (1 << 21)
+
 #define TLB_L2CLEAN_FR (1 << 29)               /* Feroceon */
 #define TLB_DCLEAN     (1 << 30)
 #define TLB_WB         (1 << 31)
@@ -176,9 +182,17 @@
 # define v6wbi_always_flags    (-1UL)
 #endif
 
+#ifdef CONFIG_SMP
+#define v7wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \
+                        TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
+#else
+#define v7wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \
+                        TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
+#endif
+
 #ifdef CONFIG_CPU_TLB_V7
-# define v7wbi_possible_flags  v6wbi_tlb_flags
-# define v7wbi_always_flags    v6wbi_tlb_flags
+# define v7wbi_possible_flags  v7wbi_tlb_flags
+# define v7wbi_always_flags    v7wbi_tlb_flags
 # ifdef _TLB
 #  define MULTI_TLB 1
 # else
@@ -316,6 +330,8 @@ static inline void local_flush_tlb_all(void)
                asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc");
        if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL))
                asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc");
+       if (tlb_flag(TLB_V7_UIS_FULL))
+               asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero) : "cc");
 
        if (tlb_flag(TLB_BTB)) {
                /* flush the branch target cache */
@@ -351,6 +367,8 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm)
                asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc");
        if (tlb_flag(TLB_V6_I_ASID))
                asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc");
+       if (tlb_flag(TLB_V7_UIS_ASID))
+               asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid) : "cc");
 
        if (tlb_flag(TLB_BTB)) {
                /* flush the branch target cache */
@@ -389,6 +407,8 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
                asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc");
        if (tlb_flag(TLB_V6_I_PAGE))
                asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc");
+       if (tlb_flag(TLB_V7_UIS_PAGE))
+               asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr) : "cc");
 
        if (tlb_flag(TLB_BTB)) {
                /* flush the branch target cache */
@@ -424,6 +444,8 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
                asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc");
        if (tlb_flag(TLB_V6_I_PAGE))
                asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc");
+       if (tlb_flag(TLB_V7_UIS_PAGE))
+               asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr) : "cc");
 
        if (tlb_flag(TLB_BTB)) {
                /* flush the branch target cache */

diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 6014dfd22af44a56270890726f63b565ac9620a4..ece658e773a6ff71075c63f40ec082439c0b4bf4 100644 (file)
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -32,6 +32,7 @@
 #include <asm/processor.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
+#include <asm/cputype.h>
 
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
@@ -545,6 +546,12 @@ struct tlb_args {
        unsigned long ta_end;
 };
 
+/* all SMP configurations have the extended CPUID registers */
+static inline int tlb_ops_need_broadcast(void)
+{
+       return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2;
+}
+
 static inline void ipi_flush_tlb_all(void *ignored)
 {
        local_flush_tlb_all();
@@ -587,51 +594,61 @@ static inline void ipi_flush_tlb_kernel_range(void *arg)
 
 void flush_tlb_all(void)
 {
-       on_each_cpu(ipi_flush_tlb_all, NULL, 1);
+       if (tlb_ops_need_broadcast())
+               on_each_cpu(ipi_flush_tlb_all, NULL, 1);
+       else
+               local_flush_tlb_all();
 }
 
 void flush_tlb_mm(struct mm_struct *mm)
 {
-       on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask);
+       if (tlb_ops_need_broadcast())
+               on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask);
+       else
+               local_flush_tlb_mm(mm);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 {
-       struct tlb_args ta;
-
-       ta.ta_vma = vma;
-       ta.ta_start = uaddr;
-
-       on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+       if (tlb_ops_need_broadcast()) {
+               struct tlb_args ta;
+               ta.ta_vma = vma;
+               ta.ta_start = uaddr;
+               on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+       } else
+               local_flush_tlb_page(vma, uaddr);
 }
 
 void flush_tlb_kernel_page(unsigned long kaddr)
 {
-       struct tlb_args ta;
-
-       ta.ta_start = kaddr;
-
-       on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
+       if (tlb_ops_need_broadcast()) {
+               struct tlb_args ta;
+               ta.ta_start = kaddr;
+               on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
+       } else
+               local_flush_tlb_kernel_page(kaddr);
 }
 
 void flush_tlb_range(struct vm_area_struct *vma,
                      unsigned long start, unsigned long end)
 {
-       struct tlb_args ta;
-
-       ta.ta_vma = vma;
-       ta.ta_start = start;
-       ta.ta_end = end;
-
-       on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+       if (tlb_ops_need_broadcast()) {
+               struct tlb_args ta;
+               ta.ta_vma = vma;
+               ta.ta_start = start;
+               ta.ta_end = end;
+               on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask);
+       } else
+               local_flush_tlb_range(vma, start, end);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
-       struct tlb_args ta;
-
-       ta.ta_start = start;
-       ta.ta_end = end;
-
-       on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
+       if (tlb_ops_need_broadcast()) {
+               struct tlb_args ta;
+               ta.ta_start = start;
+               ta.ta_end = end;
+               on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
+       } else
+               local_flush_tlb_kernel_range(start, end);
 }

diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 3397f1e64d76378d676d7c7e9f517bde86cff9ad..c3f737516836471894e7419cab4fec6d44e1b633 100644 (file)
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -176,8 +176,8 @@ cpu_v7_name:
  */
 __v7_setup:
 #ifdef CONFIG_SMP
-       mrc     p15, 0, r0, c1, c0, 1           @ Enable SMP/nAMP mode
-       orr     r0, r0, #(0x1 << 6)
+       mrc     p15, 0, r0, c1, c0, 1           @ Enable SMP/nAMP mode and
+       orr     r0, r0, #(1 << 6) | (1 << 0)    @ TLB ops broadcasting
        mcr     p15, 0, r0, c1, c0, 1
 #endif
        adr     r12, __v7_setup_stack           @ the local stack

diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S
index b637e7380ab7ed100f74ad1db3e9d31aaf15c0e5..a26a605b73bd91688417dddc00378f286bfc6a33 100644 (file)
--- a/arch/arm/mm/tlb-v7.S
+++ b/arch/arm/mm/tlb-v7.S
@@ -42,9 +42,11 @@ ENTRY(v7wbi_flush_user_tlb_range)
        mov     r1, r1, lsl #PAGE_SHIFT
        vma_vm_flags r2, r2                     @ get vma->vm_flags
 1:
-       mcr     p15, 0, r0, c8, c6, 1           @ TLB invalidate D MVA (was 1)
-       tst     r2, #VM_EXEC                    @ Executable area ?
-       mcrne   p15, 0, r0, c8, c5, 1           @ TLB invalidate I MVA (was 1)
+#ifdef CONFIG_SMP
+       mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable) 
+#else
+       mcr     p15, 0, r0, c8, c7, 1           @ TLB invalidate U MVA
+#endif
        add     r0, r0, #PAGE_SZ
        cmp     r0, r1
        blo     1b
@@ -69,8 +71,11 @@ ENTRY(v7wbi_flush_kern_tlb_range)
        mov     r0, r0, lsl #PAGE_SHIFT
        mov     r1, r1, lsl #PAGE_SHIFT
 1:
-       mcr     p15, 0, r0, c8, c6, 1           @ TLB invalidate D MVA
-       mcr     p15, 0, r0, c8, c5, 1           @ TLB invalidate I MVA
+#ifdef CONFIG_SMP
+       mcr     p15, 0, r0, c8, c3, 1           @ TLB invalidate U MVA (shareable)
+#else
+       mcr     p15, 0, r0, c8, c7, 1           @ TLB invalidate U MVA
+#endif
        add     r0, r0, #PAGE_SZ
        cmp     r0, r1
        blo     1b
@@ -87,5 +92,5 @@ ENDPROC(v7wbi_flush_kern_tlb_range)
 ENTRY(v7wbi_tlb_fns)
        .long   v7wbi_flush_user_tlb_range
        .long   v7wbi_flush_kern_tlb_range
-       .long   v6wbi_tlb_flags
+       .long   v7wbi_tlb_flags
        .size   v7wbi_tlb_fns, . - v7wbi_tlb_fns