FROMLIST: arm64: entry: Add fake CPU feature for unmapping the kernel at EL0

Allow explicit disabling of the entry trampoline on the kernel command
line (kpti=off) by adding a fake CPU feature (ARM64_UNMAP_KERNEL_AT_EL0)
that can be used to toggle the alternative sequences in our entry code and
avoid use of the trampoline altogether if desired. This also allows us to
make use of a static key in arm64_kernel_unmapped_at_el0().

Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Tested-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
(cherry picked from git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git
 commit ea1e3de85e94d711f63437c04624aa0e8de5c8b3)

Change-Id: I11cb874d12a7d0921f452c62b0752e0028a8e0a7
[ghackmann@google.com:
 - adjust context
 - apply cpucaps.h changes to cpufeature.h
 - replace cpus_have_const_cap() with cpus_have_cap()
 - tweak unmap_kernel_at_el0() declaration to match 4.4 APIs]
Signed-off-by: Greg Hackmann <ghackmann@google.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 4c56ecdf2dc8d719fa8dbaea7b3ff7bf1092ca88..2c6497a042b16c065393061ffce2df82645a9e7b 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -35,8 +35,9 @@
 #define ARM64_ALT_PAN_NOT_UAO                  10
 #define ARM64_HAS_VIRT_HOST_EXTN               11
 #define ARM64_WORKAROUND_CAVIUM_27456          12
+#define ARM64_UNMAP_KERNEL_AT_EL0              23
 
-#define ARM64_NCAPS                            13
+#define ARM64_NCAPS                            24
 
 #ifndef __ASSEMBLY__
 

diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 8e60392c9137d233e32a1518e5d59e9cbc38f63f..bb907189aef024f1cf5bd023f27317dcff929b96 100644 (file)
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -34,7 +34,8 @@ typedef struct {
 
 static inline bool arm64_kernel_unmapped_at_el0(void)
 {
-       return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0);
+       return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&
+              cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0);
 }
 
 extern void paging_init(void);

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 56cbcfba0c02ae42df39e5c3fd6fb3a72761761d..0e34add11c7a14ec219bc53dc76229f3522adcf7 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -650,6 +650,39 @@ static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry)
        return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, rv_min, rv_max);
 }
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
+
+static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry)
+{
+       /* Forced on command line? */
+       if (__kpti_forced) {
+               pr_info_once("kernel page table isolation forced %s by command line option\n",
+                            __kpti_forced > 0 ? "ON" : "OFF");
+               return __kpti_forced > 0;
+       }
+
+       /* Useful for KASLR robustness */
+       if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
+               return true;
+
+       return false;
+}
+
+static int __init parse_kpti(char *str)
+{
+       bool enabled;
+       int ret = strtobool(str, &enabled);
+
+       if (ret)
+               return ret;
+
+       __kpti_forced = enabled ? 1 : -1;
+       return 0;
+}
+__setup("kpti=", parse_kpti);
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
 static const struct arm64_cpu_capabilities arm64_features[] = {
        {
                .desc = "GIC system register CPU interface",
@@ -702,6 +735,12 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .matches = cpufeature_pan_not_uao,
        },
 #endif /* CONFIG_ARM64_PAN */
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+       {
+               .capability = ARM64_UNMAP_KERNEL_AT_EL0,
+               .matches = unmap_kernel_at_el0,
+       },
+#endif
        {},
 };
 

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 5be224cd01e7d67a8ab7ec994a67d22e5670b62a..60d69614e17465dbc10014fba079edc1e0f9d378 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -74,6 +74,7 @@
        .macro kernel_ventry, el, label, regsize = 64
        .align 7
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+alternative_if ARM64_UNMAP_KERNEL_AT_EL0
        .if     \el == 0
        .if     \regsize == 64
        mrs     x30, tpidrro_el0
@@ -82,6 +83,7 @@
        mov     x30, xzr
        .endif
        .endif
+alternative_else_nop_endif
 #endif
 
        sub     sp, sp, #S_FRAME_SIZE
@@ -287,10 +289,9 @@ alternative_else_nop_endif
        ldr     lr, [sp, #S_LR]
        add     sp, sp, #S_FRAME_SIZE           // restore sp
 
-#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
-       eret
-#else
        .if     \el == 0
+alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
        bne     4f
        msr     far_el1, x30
        tramp_alias     x30, tramp_exit_native
@@ -298,10 +299,10 @@ alternative_else_nop_endif
 4:
        tramp_alias     x30, tramp_exit_compat
        br      x30
+#endif
        .else
        eret
        .endif
-#endif
        .endm
 
        .macro  irq_stack_entry