arm64: switch to relative exception tables

Instead of using absolute addresses for both the exception location
and the fixup, use offsets relative to the exception table entry values.
Not only does this cut the size of the exception table in half, it is
also a prerequisite for KASLR, since absolute exception table entries
are subject to dynamic relocation, which is incompatible with the sorting
of the exception table that occurs at build time.

This patch also introduces the _ASM_EXTABLE preprocessor macro (which
exists on x86 as well) and its _asm_extable assembly counterpart, as
shorthands to emit exception table entries.

Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h
index a9fc24ec1aa9ee5f19909dda3d0ea465f31365ec..beccbdefa106a4ba0dd7d98537427a37d2c26517 100644 (file)
--- a/arch/arm64/include/asm/alternative.h
+++ b/arch/arm64/include/asm/alternative.h
@@ -157,11 +157,8 @@ void apply_alternatives(void *start, size_t length);
                        add     \addr, \addr, \post_inc;
                alternative_endif
 
-               .section __ex_table,"a";
-               .align  3;
-               .quad   8888b,\l;
-               .quad   8889b,\l;
-               .previous;
+               _asm_extable    8888b,\l;
+               _asm_extable    8889b,\l;
        .endm
 
        .macro uao_stp l, reg1, reg2, addr, post_inc
@@ -175,11 +172,8 @@ void apply_alternatives(void *start, size_t length);
                        add     \addr, \addr, \post_inc;
                alternative_endif
 
-               .section __ex_table,"a";
-               .align  3;
-               .quad   8888b,\l;
-               .quad   8889b,\l;
-               .previous
+               _asm_extable    8888b,\l;
+               _asm_extable    8889b,\l;
        .endm
 
        .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
@@ -191,10 +185,7 @@ void apply_alternatives(void *start, size_t length);
                        add             \addr, \addr, \post_inc;
                alternative_endif
 
-               .section __ex_table,"a";
-               .align  3;
-               .quad   8888b,\l;
-               .previous
+               _asm_extable    8888b,\l;
        .endm
 #else
        .macro uao_ldp l, reg1, reg2, addr, post_inc

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index ba5aff6c830e7122a96f61d73b1368cffefcf49e..70f7b9e04598bdf953b56d40794f1ab5aaf966ca 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -94,12 +94,19 @@
        dmb     \opt
        .endm
 
+/*
+ * Emit an entry into the exception table
+ */
+       .macro          _asm_extable, from, to
+       .pushsection    __ex_table, "a"
+       .align          3
+       .long           (\from - .), (\to - .)
+       .popsection
+       .endm
+
 #define USER(l, x...)                          \
 9999:  x;                                      \
-       .section __ex_table,"a";                \
-       .align  3;                              \
-       .quad   9999b,l;                        \
-       .previous
+       _asm_extable    9999b, l
 
 /*
  * Register aliases.

diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index 5f3ab8c1db55cca8dbf4c9e1fc315e90335b6d60..f2585cdd32c29832566718e99d7b5fd9c61d2322 100644 (file)
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -42,10 +42,8 @@
 "4:    mov     %w0, %w5\n"                                             \
 "      b       3b\n"                                                   \
 "      .popsection\n"                                                  \
-"      .pushsection __ex_table,\"a\"\n"                                \
-"      .align  3\n"                                                    \
-"      .quad   1b, 4b, 2b, 4b\n"                                       \
-"      .popsection\n"                                                  \
+       _ASM_EXTABLE(1b, 4b)                                            \
+       _ASM_EXTABLE(2b, 4b)                                            \
        ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,            \
                    CONFIG_ARM64_PAN)                                   \
        : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)       \
@@ -134,10 +132,8 @@ ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
 "4:    mov     %w0, %w6\n"
 "      b       3b\n"
 "      .popsection\n"
-"      .pushsection __ex_table,\"a\"\n"
-"      .align  3\n"
-"      .quad   1b, 4b, 2b, 4b\n"
-"      .popsection\n"
+       _ASM_EXTABLE(1b, 4b)
+       _ASM_EXTABLE(2b, 4b)
 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, CONFIG_ARM64_PAN)
        : "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
        : "r" (oldval), "r" (newval), "Ir" (-EFAULT)

diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 16ba0d5c9740c5aea0ac8d0e8748da4a92e61e89..0685d74572af788b05d44658c7dba7e4fc3742ba 100644 (file)
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -36,11 +36,11 @@
 #define VERIFY_WRITE 1
 
 /*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue.  No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
+ * The exception table consists of pairs of relative offsets: the first
+ * is the relative offset to an instruction that is allowed to fault,
+ * and the second is the relative offset at which the program should
+ * continue. No registers are modified, so it is entirely up to the
+ * continuation code to figure out what to do.
  *
  * All the routines below use bits of fixup code that are out of line
  * with the main instruction path.  This means when everything is well,
@@ -50,9 +50,11 @@
 
 struct exception_table_entry
 {
-       unsigned long insn, fixup;
+       int insn, fixup;
 };
 
+#define ARCH_HAS_RELATIVE_EXTABLE
+
 extern int fixup_exception(struct pt_regs *regs);
 
 #define KERNEL_DS      (-1UL)
@@ -115,6 +117,12 @@ static inline void set_fs(mm_segment_t fs)
 #define access_ok(type, addr, size)    __range_ok(addr, size)
 #define user_addr_max                  get_fs
 
+#define _ASM_EXTABLE(from, to)                                         \
+       "       .pushsection    __ex_table, \"a\"\n"                    \
+       "       .align          3\n"                                    \
+       "       .long           (" #from " - .), (" #to " - .)\n"       \
+       "       .popsection\n"
+
 /*
  * The "__xxx" versions of the user access functions do not verify the address
  * space - it must have been done previously with a separate "access_ok()"
@@ -134,10 +142,7 @@ static inline void set_fs(mm_segment_t fs)
        "       mov     %1, #0\n"                                       \
        "       b       2b\n"                                           \
        "       .previous\n"                                            \
-       "       .section __ex_table,\"a\"\n"                            \
-       "       .align  3\n"                                            \
-       "       .quad   1b, 3b\n"                                       \
-       "       .previous"                                              \
+       _ASM_EXTABLE(1b, 3b)                                            \
        : "+r" (err), "=&r" (x)                                         \
        : "r" (addr), "i" (-EFAULT))
 
@@ -206,10 +211,7 @@ do {                                                                       \
        "3:     mov     %w0, %3\n"                                      \
        "       b       2b\n"                                           \
        "       .previous\n"                                            \
-       "       .section __ex_table,\"a\"\n"                            \
-       "       .align  3\n"                                            \
-       "       .quad   1b, 3b\n"                                       \
-       "       .previous"                                              \
+       _ASM_EXTABLE(1b, 3b)                                            \
        : "+r" (err)                                                    \
        : "r" (x), "r" (addr), "i" (-EFAULT))
 

diff --git a/arch/arm64/include/asm/word-at-a-time.h b/arch/arm64/include/asm/word-at-a-time.h
index aab5bf09e9d902f7bdf3d09e61fcd97db748c505..2b79b8a89457bd70e20e24ba33c8029195e3284c 100644 (file)
--- a/arch/arm64/include/asm/word-at-a-time.h
+++ b/arch/arm64/include/asm/word-at-a-time.h
@@ -16,6 +16,8 @@
 #ifndef __ASM_WORD_AT_A_TIME_H
 #define __ASM_WORD_AT_A_TIME_H
 
+#include <asm/uaccess.h>
+
 #ifndef __AARCH64EB__
 
 #include <linux/kernel.h>
@@ -81,10 +83,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr)
 #endif
        "       b       2b\n"
        "       .popsection\n"
-       "       .pushsection __ex_table,\"a\"\n"
-       "       .align  3\n"
-       "       .quad   1b, 3b\n"
-       "       .popsection"
+       _ASM_EXTABLE(1b, 3b)
        : "=&r" (ret), "=&r" (offset)
        : "r" (addr), "Q" (*(unsigned long *)addr));
 

diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index 3e01207917b13db999b08d36cb722c153ad8cd7f..c37202c0c838d01a71d56b05d114cd0b419ff480 100644 (file)
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -297,11 +297,8 @@ static void __init register_insn_emulation_sysctl(struct ctl_table *table)
        "4:     mov             %w0, %w5\n"                     \
        "       b               3b\n"                           \
        "       .popsection"                                    \
-       "       .pushsection     __ex_table,\"a\"\n"            \
-       "       .align          3\n"                            \
-       "       .quad           0b, 4b\n"                       \
-       "       .quad           1b, 4b\n"                       \
-       "       .popsection\n"                                  \
+       _ASM_EXTABLE(0b, 4b)                                    \
+       _ASM_EXTABLE(1b, 4b)                                    \
        ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,    \
                CONFIG_ARM64_PAN)                               \
        : "=&r" (res), "+r" (data), "=&r" (temp)                \

diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c
index 79444279ba8c674316e34cfe0861c42021fa92f3..81acd4706878f85d8821f0ff924bff05adc31c97 100644 (file)
--- a/arch/arm64/mm/extable.c
+++ b/arch/arm64/mm/extable.c
@@ -11,7 +11,7 @@ int fixup_exception(struct pt_regs *regs)
 
        fixup = search_exception_tables(instruction_pointer(regs));
        if (fixup)
-               regs->pc = fixup->fixup;
+               regs->pc = (unsigned long)&fixup->fixup + fixup->fixup;
 
        return fixup != NULL;
 }

diff --git a/scripts/sortextable.c b/scripts/sortextable.c
index ecefa0a634f88182aca8ec82ed523e40baafa2d8..19d83647846cf5213157e5d99fab9ed8a8a70757 100644 (file)
--- a/scripts/sortextable.c
+++ b/scripts/sortextable.c
@@ -282,12 +282,12 @@ do_file(char const *const fname)
        case EM_386:
        case EM_X86_64:
        case EM_S390:
+       case EM_AARCH64:
                custom_sort = sort_relative_table;
                break;
        case EM_ARCOMPACT:
        case EM_ARCV2:
        case EM_ARM:
-       case EM_AARCH64:
        case EM_MICROBLAZE:
        case EM_MIPS:
        case EM_XTENSA: