FROMLIST: arm64: Introduce uaccess_{disable,enable} functionality based on TTBR0_EL1

This patch adds the uaccess macros/functions to disable access to user
space by setting TTBR0_EL1 to a reserved zeroed page. Since the value
written to TTBR0_EL1 must be a physical address, for simplicity this
patch introduces a reserved_ttbr0 page at a constant offset from
swapper_pg_dir. The uaccess_disable code uses the ttbr1_el1 value
adjusted by the reserved_ttbr0 offset.

Enabling access to user is done by restoring TTBR0_EL1 with the value
from the struct thread_info ttbr0 variable. Interrupts must be disabled
during the uaccess_ttbr0_enable code to ensure the atomicity of the
thread_info.ttbr0 read and TTBR0_EL1 write. This patch also moves the
get_thread_info asm macro from entry.S to assembler.h for reuse in the
uaccess_ttbr0_* macros.

Cc: Will Deacon <will.deacon@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Change-Id: Idf09a870b8612dce23215bce90d88781f0c0c3aa
(cherry picked from commit 940d37234182d2675ab8ab46084840212d735018)
Signed-off-by: Sami Tolvanen <samitolvanen@google.com>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 9d3e77a5cf07546fbd024c2f21615ea4a20f0880..aeb4554b3af386f215ba9fbe75f0be671b13d8eb 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -53,6 +53,15 @@
        msr     daifclr, #2
        .endm
 
+       .macro  save_and_disable_irq, flags
+       mrs     \flags, daif
+       msr     daifset, #2
+       .endm
+
+       .macro  restore_irq, flags
+       msr     daif, \flags
+       .endm
+
 /*
  * Enable and disable debug exceptions.
  */
@@ -362,6 +371,13 @@ alternative_endif
        movk    \reg, :abs_g0_nc:\val
        .endm
 
+/*
+ * Return the current thread_info.
+ */
+       .macro  get_thread_info, rd
+       mrs     \rd, sp_el0
+       .endm
+
 /*
  * Errata workaround post TTBR0_EL1 update.
  */

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 727e594ac5c25d150cfc777e9a490eb60db08d19..f125c03ab2e1d1b9bb17d12acfe151b678fc9a03 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -188,6 +188,12 @@ static inline bool system_supports_mixed_endian_el0(void)
        return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1));
 }
 
+static inline bool system_uses_ttbr0_pan(void)
+{
+       return IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN) &&
+               !cpus_have_cap(ARM64_HAS_PAN);
+}
+
 #endif /* __ASSEMBLY__ */
 
 #endif

diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 7e51d1b57c0c56461a9be0fb825fa84a9faeea8b..7803343e5881fbd7b2f635b25082d3e91d2583f8 100644 (file)
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -19,6 +19,7 @@
 #ifndef __ASM_KERNEL_PGTABLE_H
 #define __ASM_KERNEL_PGTABLE_H
 
+#include <asm/pgtable.h>
 #include <asm/sparsemem.h>
 
 /*
@@ -54,6 +55,12 @@
 #define SWAPPER_DIR_SIZE       (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
 #define IDMAP_DIR_SIZE         (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
 
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+#define RESERVED_TTBR0_SIZE    (PAGE_SIZE)
+#else
+#define RESERVED_TTBR0_SIZE    (0)
+#endif
+
 /* Initial memory map size */
 #if ARM64_SWAPPER_USES_SECTION_MAPS
 #define SWAPPER_BLOCK_SHIFT    SECTION_SHIFT

diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index abd64bd1f6d9f0160a3122555cf23be1a30f87eb..b3325a9cb90fda6c9fb052e560a351d469c83fda 100644 (file)
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -47,6 +47,9 @@ typedef unsigned long mm_segment_t;
 struct thread_info {
        unsigned long           flags;          /* low level flags */
        mm_segment_t            addr_limit;     /* address limit */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+       u64                     ttbr0;          /* saved TTBR0_EL1 */
+#endif
        struct task_struct      *task;          /* main task structure */
        int                     preempt_count;  /* 0 => preemptable, <0 => bug */
        int                     cpu;            /* cpu */

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index c8ef22a9a83bfb7d543c8ae9462e9ca0379db05d..c37c064d7cddd62ab5c3334ff89e3077db3ffa91 100644 (file)
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -28,6 +28,7 @@
 
 #include <asm/alternative.h>
 #include <asm/cpufeature.h>
+#include <asm/kernel-pgtable.h>
 #include <asm/ptrace.h>
 #include <asm/sysreg.h>
 #include <asm/errno.h>
@@ -128,16 +129,57 @@ static inline void set_fs(mm_segment_t fs)
 /*
  * User access enabling/disabling.
  */
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+static inline void uaccess_ttbr0_disable(void)
+{
+       unsigned long ttbr;
+
+       /* reserved_ttbr0 placed at the end of swapper_pg_dir */
+       ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
+       write_sysreg(ttbr, ttbr0_el1);
+       isb();
+}
+
+static inline void uaccess_ttbr0_enable(void)
+{
+       unsigned long flags;
+
+       /*
+        * Disable interrupts to avoid preemption between reading the 'ttbr0'
+        * variable and the MSR. A context switch could trigger an ASID
+        * roll-over and an update of 'ttbr0'.
+        */
+       local_irq_save(flags);
+       write_sysreg(current_thread_info()->ttbr0, ttbr0_el1);
+       isb();
+       local_irq_restore(flags);
+}
+#else
+static inline void uaccess_ttbr0_disable(void)
+{
+}
+
+static inline void uaccess_ttbr0_enable(void)
+{
+}
+#endif
+
 #define __uaccess_disable(alt)                                         \
 do {                                                                   \
-       asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,                  \
-                       CONFIG_ARM64_PAN));                             \
+       if (system_uses_ttbr0_pan())                                    \
+               uaccess_ttbr0_disable();                                \
+       else                                                            \
+               asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt,          \
+                               CONFIG_ARM64_PAN));                     \
 } while (0)
 
 #define __uaccess_enable(alt)                                          \
 do {                                                                   \
-       asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,                  \
-                       CONFIG_ARM64_PAN));                             \
+       if (system_uses_ttbr0_pan())                                    \
+               uaccess_ttbr0_enable();                                 \
+       else                                                            \
+               asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt,          \
+                               CONFIG_ARM64_PAN));                     \
 } while (0)
 
 static inline void uaccess_disable(void)
@@ -367,12 +409,39 @@ extern __must_check long strnlen_user(const char __user *str, long n);
 
 #include <asm/alternative.h>
 #include <asm/assembler.h>
+#include <asm/kernel-pgtable.h>
+
+/*
+ * User access enabling/disabling macros.
+ */
+       .macro  uaccess_ttbr0_disable, tmp1
+       mrs     \tmp1, ttbr1_el1                // swapper_pg_dir
+       add     \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir
+       msr     ttbr0_el1, \tmp1                // set reserved TTBR0_EL1
+       isb
+       .endm
+
+       .macro  uaccess_ttbr0_enable, tmp1
+       get_thread_info \tmp1
+       ldr     \tmp1, [\tmp1, #TI_TTBR0]       // load saved TTBR0_EL1
+       msr     ttbr0_el1, \tmp1                // set the non-PAN TTBR0_EL1
+       isb
+       .endm
 
 /*
- * User access enabling/disabling macros. These are no-ops when UAO is
- * present.
+ * These macros are no-ops when UAO is present.
  */
        .macro  uaccess_disable_not_uao, tmp1
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+alternative_if_not ARM64_HAS_PAN
+       uaccess_ttbr0_disable \tmp1
+alternative_else
+       nop
+       nop
+       nop
+       nop
+alternative_endif
+#endif
 alternative_if_not ARM64_ALT_PAN_NOT_UAO
        nop
 alternative_else
@@ -381,6 +450,21 @@ alternative_endif
        .endm
 
        .macro  uaccess_enable_not_uao, tmp1, tmp2
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+alternative_if_not ARM64_HAS_PAN
+       save_and_disable_irq \tmp2              // avoid preemption
+       uaccess_ttbr0_enable \tmp1
+       restore_irq \tmp2
+alternative_else
+       nop
+       nop
+       nop
+       nop
+       nop
+       nop
+       nop
+alternative_endif
+#endif
 alternative_if_not ARM64_ALT_PAN_NOT_UAO
        nop
 alternative_else

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 087cf9a65359b5fac32a0fcb4c5fa2804ffd73bc..d0ec987dba5bec33491eaed671c02232a4faadcb 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -36,6 +36,9 @@ int main(void)
   DEFINE(TI_FLAGS,             offsetof(struct thread_info, flags));
   DEFINE(TI_PREEMPT,           offsetof(struct thread_info, preempt_count));
   DEFINE(TI_ADDR_LIMIT,                offsetof(struct thread_info, addr_limit));
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+  DEFINE(TI_TTBR0,             offsetof(struct thread_info, ttbr0));
+#endif
   DEFINE(TI_TASK,              offsetof(struct thread_info, task));
   DEFINE(TI_CPU,               offsetof(struct thread_info, cpu));
   BLANK();

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 7566cad9fa1da5a882ada85b5801a8ae33f4da0c..40ee3f2933e78dffe011e4d2e5615f0d42222e53 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -43,6 +43,7 @@ unsigned int compat_elf_hwcap2 __read_mostly;
 #endif
 
 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
+EXPORT_SYMBOL(cpu_hwcaps);
 
 #define __ARM64_FTR_BITS(SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
        {                                               \

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 533e1c9fd5a6e01cc8cd4696fe17d9d279b06f07..8eb8eb085036d3a85297c75c779b464bef2a95f7 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -190,10 +190,6 @@ alternative_endif
        eret                                    // return to kernel
        .endm
 
-       .macro  get_thread_info, rd
-       mrs     \rd, sp_el0
-       .endm
-
        .macro  irq_stack_entry
        mov     x19, sp                 // preserve the original sp
 

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 54fd1e2590a27831f8b2a659a0c053bf1c638ce0..9bfa58fea8ce315e224d1f1115a62f063da15d59 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -321,14 +321,14 @@ __create_page_tables:
         * dirty cache lines being evicted.
         */
        mov     x0, x25
-       add     x1, x26, #SWAPPER_DIR_SIZE
+       add     x1, x26, #SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
        bl      __inval_cache_range
 
        /*
         * Clear the idmap and swapper page tables.
         */
        mov     x0, x25
-       add     x6, x26, #SWAPPER_DIR_SIZE
+       add     x6, x26, #SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
 1:     stp     xzr, xzr, [x0], #16
        stp     xzr, xzr, [x0], #16
        stp     xzr, xzr, [x0], #16
@@ -406,7 +406,7 @@ __create_page_tables:
         * tables again to remove any speculatively loaded cache lines.
         */
        mov     x0, x25
-       add     x1, x26, #SWAPPER_DIR_SIZE
+       add     x1, x26, #SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE
        dmb     sy
        bl      __inval_cache_range
 

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 14813a6ca13b985d3409d37b9a7fd3c38506377a..87fd0556a1bd8330b3610308cc3fffd61a28b32e 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -185,6 +185,11 @@ SECTIONS
        swapper_pg_dir = .;
        . += SWAPPER_DIR_SIZE;
 
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+       reserved_ttbr0 = .;
+       . += RESERVED_TTBR0_SIZE;
+#endif
+
        _end = .;
 
        STABS_DEBUG