arm64: Work around systems with mismatched cache line sizes

Systems with differing CPU i-cache/d-cache line sizes can cause
problems with the cache management by software when the execution
is migrated from one to another. Usually, the application reads
the cache size on a CPU and then uses that length to perform cache
operations. However, if it gets migrated to another CPU with a smaller
cache line size, things could go completely wrong. To prevent such
cases, always use the smallest cache line size among the CPUs. The
kernel CPU feature infrastructure already keeps track of the safe
value for all CPUID registers including CTR. This patch works around
the problem by :

For kernel, dynamically patch the kernel to read the cache size
from the system wide copy of CTR_EL0.

For applications, trap read accesses to CTR_EL0 (by clearing the SCTLR.UCT)
and emulate the mrs instruction to return the system wide safe value
of CTR_EL0.

For faster access (i.e, avoiding to lookup the system wide value of CTR_EL0
via read_system_reg), we keep track of the pointer to table entry for
CTR_EL0 in the CPU feature infrastructure.

Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Andre Przywara <andre.przywara@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index a4bb3f52d9ef1e0e437687a6d41f2dd13b907575..f09a5ae48a44a007f6e1f4af713fab836fb7c6f5 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -216,6 +216,20 @@ lr .req    x30             // link register
        .macro  mmid, rd, rn
        ldr     \rd, [\rn, #MM_CONTEXT_ID]
        .endm
+/*
+ * read_ctr - read CTR_EL0. If the system has mismatched
+ * cache line sizes, provide the system wide safe value
+ * from arm64_ftr_reg_ctrel0.sys_val
+ */
+       .macro  read_ctr, reg
+alternative_if_not ARM64_MISMATCHED_CACHE_LINE_SIZE
+       mrs     \reg, ctr_el0                   // read CTR
+       nop
+alternative_else
+       ldr_l   \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
+alternative_endif
+       .endm
+
 
 /*
  * raw_dcache_line_size - get the minimum D-cache line size on this CPU
@@ -232,7 +246,10 @@ lr .req    x30             // link register
  * dcache_line_size - get the safe D-cache line size across all CPUs
  */
        .macro  dcache_line_size, reg, tmp
-       raw_dcache_line_size    \reg, \tmp
+       read_ctr        \tmp
+       ubfm            \tmp, \tmp, #16, #19    // cache line size encoding
+       mov             \reg, #4                // bytes per word
+       lsl             \reg, \reg, \tmp        // actual cache line size
        .endm
 
 /*
@@ -250,7 +267,10 @@ lr .req    x30             // link register
  * icache_line_size - get the safe I-cache line size across all CPUs
  */
        .macro  icache_line_size, reg, tmp
-       raw_icache_line_size    \reg, \tmp
+       read_ctr        \tmp
+       and             \tmp, \tmp, #0xf        // cache line size encoding
+       mov             \reg, #4                // bytes per word
+       lsl             \reg, \reg, \tmp        // actual cache line size
        .endm
 
 /*

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 6806b86ab791152ab5ff39e729c840ee9ede0d72..758d74fedfad9bafe86835a56272deebf705e591 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -39,8 +39,9 @@
 #define ARM64_WORKAROUND_CAVIUM_27456          12
 #define ARM64_HAS_32BIT_EL0                    13
 #define ARM64_HYP_OFFSET_LOW                   14
+#define ARM64_MISMATCHED_CACHE_LINE_SIZE       15
 
-#define ARM64_NCAPS                            15
+#define ARM64_NCAPS                            16
 
 #ifndef __ASSEMBLY__
 

diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 9875b326a73e07a2ef347e00b29c48cdeb7b997d..d14c478976d0abeb435163da5c16599a0f2c3b53 100644 (file)
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -149,6 +149,9 @@
                                         ((op2) << ESR_ELx_SYS64_ISS_OP2_SHIFT) | \
                                         ((crn) << ESR_ELx_SYS64_ISS_CRN_SHIFT) | \
                                         ((crm) << ESR_ELx_SYS64_ISS_CRM_SHIFT))
+
+#define ESR_ELx_SYS64_ISS_SYS_OP_MASK  (ESR_ELx_SYS64_ISS_SYS_MASK | \
+                                        ESR_ELx_SYS64_ISS_DIR_MASK)
 /*
  * User space cache operations have the following sysreg encoding
  * in System instructions.
@@ -167,6 +170,11 @@
 #define ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL \
                                (ESR_ELx_SYS64_ISS_SYS_VAL(1, 3, 1, 7, 0) | \
                                 ESR_ELx_SYS64_ISS_DIR_WRITE)
+
+#define ESR_ELx_SYS64_ISS_SYS_CTR      ESR_ELx_SYS64_ISS_SYS_VAL(3, 3, 1, 0, 0)
+#define ESR_ELx_SYS64_ISS_SYS_CTR_READ (ESR_ELx_SYS64_ISS_SYS_CTR | \
+                                        ESR_ELx_SYS64_ISS_DIR_READ)
+
 #ifndef __ASSEMBLY__
 #include <asm/types.h>
 

diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index e91aef2bb33db8adb96c35febe6cb4202ef3d38e..7e4ecd1d2ac9c29ee250776a3415d9330d014a42 100644 (file)
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -100,6 +100,7 @@
 /* SCTLR_EL1 specific flags. */
 #define SCTLR_EL1_UCI          (1 << 26)
 #define SCTLR_EL1_SPAN         (1 << 23)
+#define SCTLR_EL1_UCT          (1 << 15)
 #define SCTLR_EL1_SED          (1 << 8)
 #define SCTLR_EL1_CP15BEN      (1 << 5)
 

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 05070b72fc28c5f27d6c943d3de0ee4518b12d3f..4a2f0f0fef329d81b272c3d7430ee7607cec6d84 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -23,6 +23,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/kvm_host.h>
 #include <linux/suspend.h>
+#include <asm/cpufeature.h>
 #include <asm/thread_info.h>
 #include <asm/memory.h>
 #include <asm/smp_plat.h>
@@ -145,5 +146,6 @@ int main(void)
   DEFINE(HIBERN_PBE_ORIG,      offsetof(struct pbe, orig_address));
   DEFINE(HIBERN_PBE_ADDR,      offsetof(struct pbe, address));
   DEFINE(HIBERN_PBE_NEXT,      offsetof(struct pbe, next));
+  DEFINE(ARM64_FTR_SYSVAL,     offsetof(struct arm64_ftr_reg, sys_val));
   return 0;
 }

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 5836b3df0094199c2d101edcd0423f071edc2117..0150394f4cabf2f34b27c88ee6575b0a9b7b4489 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -30,6 +30,21 @@ is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
                                       entry->midr_range_max);
 }
 
+static bool
+has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
+                               int scope)
+{
+       WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
+       return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
+               (arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
+}
+
+static void cpu_enable_trap_ctr_access(void *__unused)
+{
+       /* Clear SCTLR_EL1.UCT */
+       config_sctlr_el1(SCTLR_EL1_UCT, 0);
+}
+
 #define MIDR_RANGE(model, min, max) \
        .def_scope = SCOPE_LOCAL_CPU, \
        .matches = is_affected_midr_range, \
@@ -107,6 +122,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                MIDR_RANGE(MIDR_THUNDERX_81XX, 0x00, 0x00),
        },
 #endif
+       {
+               .desc = "Mismatched cache line size",
+               .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
+               .matches = has_mismatched_cache_line_size,
+               .def_scope = SCOPE_LOCAL_CPU,
+               .enable = cpu_enable_trap_ctr_access,
+       },
        {
        }
 };

diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 224f64eddd93ad8ff97fb48ae60bbce69bc8d05f..93445f8b530c5e6f9d012ff0db1d96fa09093bde 100644 (file)
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -480,6 +480,14 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
                regs->pc += 4;
 }
 
+static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
+{
+       int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
+
+       regs->regs[rt] = arm64_ftr_reg_ctrel0.sys_val;
+       regs->pc += 4;
+}
+
 struct sys64_hook {
        unsigned int esr_mask;
        unsigned int esr_val;
@@ -492,6 +500,12 @@ static struct sys64_hook sys64_hooks[] = {
                .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
                .handler = user_cache_maint_handler,
        },
+       {
+               /* Trap read access to CTR_EL0 */
+               .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
+               .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
+               .handler = ctr_read_handler,
+       },
        {},
 };