ARM: 8129/1: errata: work around Cortex-A15 erratum 830321 using dummy strex

On revisions of Cortex-A15 prior to r3p3, a CLREX instruction at PL1 may
falsely trigger a watchpoint exception, leading to potential data aborts
during exception return and/or livelock.

This patch resolves the issue in the following ways:

  - Replacing our uses of CLREX with a dummy STREX sequence instead (as
    we did for v6 CPUs).

  - Removing the clrex code from v7_exit_coherency_flush and derivatives,
    since this only exists as a minor performance improvement when
    non-cached exclusives are in use (Linux doesn't use these).

Benchmarking on a variety of ARM cores revealed no measurable
performance difference with this change applied, so the change is
performed unconditionally and no new Kconfig entry is added.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Cc: stable@vger.kernel.org
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index fd43f7f55b701f95393af71be7ef2259139f2586..79ecb4f34ffb34f25037f829a6227ea69b7a9fc8 100644 (file)
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -472,7 +472,6 @@ static inline void __sync_cache_range_r(volatile void *p, size_t size)
        "mcr    p15, 0, r0, c1, c0, 0   @ set SCTLR \n\t" \
        "isb    \n\t" \
        "bl     v7_flush_dcache_"__stringify(level)" \n\t" \
-       "clrex  \n\t" \
        "mrc    p15, 0, r0, c1, c0, 1   @ get ACTLR \n\t" \
        "bic    r0, r0, #(1 << 6)       @ disable local coherency \n\t" \
        "mcr    p15, 0, r0, c1, c0, 1   @ set ACTLR \n\t" \

diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
index 8db307d0954bb03ee95550471dcc0e6f86ae560f..2fdf8679b46e19d1d9e9b0b9c3d196b7c0edbccb 100644 (file)
--- a/arch/arm/kernel/entry-header.S
+++ b/arch/arm/kernel/entry-header.S
@@ -208,26 +208,21 @@
 #endif
        .endif
        msr     spsr_cxsf, \rpsr
-#if defined(CONFIG_CPU_V6)
-       ldr     r0, [sp]
-       strex   r1, r2, [sp]                    @ clear the exclusive monitor
-       ldmib   sp, {r1 - pc}^                  @ load r1 - pc, cpsr
-#elif defined(CONFIG_CPU_32v6K)
-       clrex                                   @ clear the exclusive monitor
-       ldmia   sp, {r0 - pc}^                  @ load r0 - pc, cpsr
-#else
-       ldmia   sp, {r0 - pc}^                  @ load r0 - pc, cpsr
+#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
+       @ We must avoid clrex due to Cortex-A15 erratum #830321
+       sub     r0, sp, #4                      @ uninhabited address
+       strex   r1, r2, [r0]                    @ clear the exclusive monitor
 #endif
+       ldmia   sp, {r0 - pc}^                  @ load r0 - pc, cpsr
        .endm
 
        .macro  restore_user_regs, fast = 0, offset = 0
        ldr     r1, [sp, #\offset + S_PSR]      @ get calling cpsr
        ldr     lr, [sp, #\offset + S_PC]!      @ get pc
        msr     spsr_cxsf, r1                   @ save in spsr_svc
-#if defined(CONFIG_CPU_V6)
+#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K)
+       @ We must avoid clrex due to Cortex-A15 erratum #830321
        strex   r1, r2, [sp]                    @ clear the exclusive monitor
-#elif defined(CONFIG_CPU_32v6K)
-       clrex                                   @ clear the exclusive monitor
 #endif
        .if     \fast
        ldmdb   sp, {r1 - lr}^                  @ get calling r1 - lr
@@ -261,7 +256,10 @@
        .endif
        ldr     lr, [sp, #S_SP]                 @ top of the stack
        ldrd    r0, r1, [sp, #S_LR]             @ calling lr and pc
-       clrex                                   @ clear the exclusive monitor
+
+       @ We must avoid clrex due to Cortex-A15 erratum #830321
+       strex   r2, r1, [sp, #S_LR]             @ clear the exclusive monitor
+
        stmdb   lr!, {r0, r1, \rpsr}            @ calling lr and rfe context
        ldmia   sp, {r0 - r12}
        mov     sp, lr
@@ -282,13 +280,16 @@
        .endm
 #else  /* ifdef CONFIG_CPU_V7M */
        .macro  restore_user_regs, fast = 0, offset = 0
-       clrex                                   @ clear the exclusive monitor
        mov     r2, sp
        load_user_sp_lr r2, r3, \offset + S_SP  @ calling sp, lr
        ldr     r1, [sp, #\offset + S_PSR]      @ get calling cpsr
        ldr     lr, [sp, #\offset + S_PC]       @ get pc
        add     sp, sp, #\offset + S_SP
        msr     spsr_cxsf, r1                   @ save in spsr_svc
+
+       @ We must avoid clrex due to Cortex-A15 erratum #830321
+       strex   r1, r2, [sp]                    @ clear the exclusive monitor
+
        .if     \fast
        ldmdb   sp, {r1 - r12}                  @ get calling r1 - r12
        .else

diff --git a/arch/arm/mach-exynos/mcpm-exynos.c b/arch/arm/mach-exynos/mcpm-exynos.c
index b2f8b60cf0e9035c0b8eabb4ef6fae098c8bcbf7..dc9a764a7c371d9b1ad5f27e14038cdbeca8690f 100644 (file)
--- a/arch/arm/mach-exynos/mcpm-exynos.c
+++ b/arch/arm/mach-exynos/mcpm-exynos.c
@@ -43,7 +43,6 @@
        "mcr    p15, 0, r0, c1, c0, 0   @ set SCTLR\n\t" \
        "isb\n\t"\
        "bl     v7_flush_dcache_"__stringify(level)"\n\t" \
-       "clrex\n\t"\
        "mrc    p15, 0, r0, c1, c0, 1   @ get ACTLR\n\t" \
        "bic    r0, r0, #(1 << 6)       @ disable local coherency\n\t" \
        /* Dummy Load of a device register to avoid Erratum 799270 */ \