ARM: 7684/1: errata: Workaround for Cortex-A15 erratum 798181 (TLBI/DSB operations)

On Cortex-A15 (r0p0..r3p2) the TLBI/DSB are not adequately shooting down
all use of the old entries. This patch implements the erratum workaround
which consists of:

1. Dummy TLBIMVAIS and DSB on the CPU doing the TLBI operation.
2. Send IPI to the CPUs that are running the same mm (and ASID) as the
   one being invalidated (or all the online CPUs for global pages).
3. CPU receiving the IPI executes a DMB and CLREX (part of the exception
   return code already).

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 12ea3b3d49a9eaf1b1317823c98945ec30921ab0..1cacda426a0ea6699528dd0eeedf83032825e09e 100644 (file)
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1439,6 +1439,16 @@ config ARM_ERRATA_775420
         to deadlock. This workaround puts DSB before executing ISB if
         an abort may occur on cache maintenance.
 
+config ARM_ERRATA_798181
+       bool "ARM errata: TLBI/DSB failure on Cortex-A15"
+       depends on CPU_V7 && SMP
+       help
+         On Cortex-A15 (r0p0..r3p2) the TLBI*IS/DSB operations are not
+         adequately shooting down all use of the old entries. This
+         option enables the Linux kernel workaround for this erratum
+         which sends an IPI to the CPUs that are running the same ASID
+         as the one being invalidated.
+
 endmenu
 
 source "arch/arm/common/Kconfig"

diff --git a/arch/arm/include/asm/highmem.h b/arch/arm/include/asm/highmem.h
index 8c5e828f484dd7a039a2c5c9d060d6bba008c0ef..91b99abe7a95c114be0d3b628fb8b8d09f781c74 100644 (file)
--- a/arch/arm/include/asm/highmem.h
+++ b/arch/arm/include/asm/highmem.h
@@ -41,6 +41,13 @@ extern void kunmap_high(struct page *page);
 #endif
 #endif
 
+/*
+ * Needed to be able to broadcast the TLB invalidation for kmap.
+ */
+#ifdef CONFIG_ARM_ERRATA_798181
+#undef ARCH_NEEDS_KMAP_HIGH_GET
+#endif
+
 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
 extern void *kmap_high_get(struct page *page);
 #else

diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h
index 863a6611323c70077a9428b198a10d59758ff919..a7b85e0d0cc154a90a2efadca763cc60d95e82d8 100644 (file)
--- a/arch/arm/include/asm/mmu_context.h
+++ b/arch/arm/include/asm/mmu_context.h
@@ -27,6 +27,8 @@ void __check_vmalloc_seq(struct mm_struct *mm);
 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk);
 #define init_new_context(tsk,mm)       ({ atomic64_set(&mm->context.id, 0); 0; })
 
+DECLARE_PER_CPU(atomic64_t, active_asids);
+
 #else  /* !CONFIG_CPU_HAS_ASID */
 
 #ifdef CONFIG_MMU

diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h
index 4db8c8820f0d1c832bf9efd5e81a69d32b4fc7d7..9e9c041358ca8789e4a6798aaf871007795e8fb9 100644 (file)
--- a/arch/arm/include/asm/tlbflush.h
+++ b/arch/arm/include/asm/tlbflush.h
@@ -450,6 +450,21 @@ static inline void local_flush_bp_all(void)
                isb();
 }
 
+#ifdef CONFIG_ARM_ERRATA_798181
+static inline void dummy_flush_tlb_a15_erratum(void)
+{
+       /*
+        * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
+        */
+       asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
+       dsb();
+}
+#else
+static inline void dummy_flush_tlb_a15_erratum(void)
+{
+}
+#endif
+
 /*
  *     flush_pmd_entry
  *

diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c
index bd0300531399e5eeb066a45f3c29b0d897c23b42..e82e1d24877227ba65ab716dc6c87e110617bd79 100644 (file)
--- a/arch/arm/kernel/smp_tlb.c
+++ b/arch/arm/kernel/smp_tlb.c
@@ -12,6 +12,7 @@
 
 #include <asm/smp_plat.h>
 #include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
 
 /**********************************************************************/
 
@@ -69,12 +70,72 @@ static inline void ipi_flush_bp_all(void *ignored)
        local_flush_bp_all();
 }
 
+#ifdef CONFIG_ARM_ERRATA_798181
+static int erratum_a15_798181(void)
+{
+       unsigned int midr = read_cpuid_id();
+
+       /* Cortex-A15 r0p0..r3p2 affected */
+       if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
+               return 0;
+       return 1;
+}
+#else
+static int erratum_a15_798181(void)
+{
+       return 0;
+}
+#endif
+
+static void ipi_flush_tlb_a15_erratum(void *arg)
+{
+       dmb();
+}
+
+static void broadcast_tlb_a15_erratum(void)
+{
+       if (!erratum_a15_798181())
+               return;
+
+       dummy_flush_tlb_a15_erratum();
+       smp_call_function_many(cpu_online_mask, ipi_flush_tlb_a15_erratum,
+                              NULL, 1);
+}
+
+static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
+{
+       int cpu;
+       cpumask_t mask = { CPU_BITS_NONE };
+
+       if (!erratum_a15_798181())
+               return;
+
+       dummy_flush_tlb_a15_erratum();
+       for_each_online_cpu(cpu) {
+               if (cpu == smp_processor_id())
+                       continue;
+               /*
+                * We only need to send an IPI if the other CPUs are running
+                * the same ASID as the one being invalidated. There is no
+                * need for locking around the active_asids check since the
+                * switch_mm() function has at least one dmb() (as required by
+                * this workaround) in case a context switch happens on
+                * another CPU after the condition below.
+                */
+               if (atomic64_read(&mm->context.id) ==
+                   atomic64_read(&per_cpu(active_asids, cpu)))
+                       cpumask_set_cpu(cpu, &mask);
+       }
+       smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
+}
+
 void flush_tlb_all(void)
 {
        if (tlb_ops_need_broadcast())
                on_each_cpu(ipi_flush_tlb_all, NULL, 1);
        else
                local_flush_tlb_all();
+       broadcast_tlb_a15_erratum();
 }
 
 void flush_tlb_mm(struct mm_struct *mm)
@@ -83,6 +144,7 @@ void flush_tlb_mm(struct mm_struct *mm)
                on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
        else
                local_flush_tlb_mm(mm);
+       broadcast_tlb_mm_a15_erratum(mm);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
@@ -95,6 +157,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
                                        &ta, 1);
        } else
                local_flush_tlb_page(vma, uaddr);
+       broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 }
 
 void flush_tlb_kernel_page(unsigned long kaddr)
@@ -105,6 +168,7 @@ void flush_tlb_kernel_page(unsigned long kaddr)
                on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
        } else
                local_flush_tlb_kernel_page(kaddr);
+       broadcast_tlb_a15_erratum();
 }
 
 void flush_tlb_range(struct vm_area_struct *vma,
@@ -119,6 +183,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
                                        &ta, 1);
        } else
                local_flush_tlb_range(vma, start, end);
+       broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -130,6 +195,7 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
                on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
        } else
                local_flush_tlb_kernel_range(start, end);
+       broadcast_tlb_a15_erratum();
 }
 
 void flush_bp_all(void)

diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index a5a4b2bc42ba353e7e0ae94461cf4d8691b2b68a..2ac37372ef52f4ba4db642d39bef349798f1785a 100644 (file)
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -48,7 +48,7 @@ static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
 static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
 static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
 
-static DEFINE_PER_CPU(atomic64_t, active_asids);
+DEFINE_PER_CPU(atomic64_t, active_asids);
 static DEFINE_PER_CPU(u64, reserved_asids);
 static cpumask_t tlb_flush_pending;
 
@@ -215,6 +215,7 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
        if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
                local_flush_bp_all();
                local_flush_tlb_all();
+               dummy_flush_tlb_a15_erratum();
        }
 
        atomic64_set(&per_cpu(active_asids, cpu), asid);