s390/exceptions: switch to relative exception table entries

This is the s390 port of 70627654 "x86, extable: Switch to relative
exception table entries".
Reduces the size of our exception tables by 50% on 64 bit builds.

Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 7e81ff17a89bc74afe2ee42626f400270f6e408c..f3e0aabfc6bcb003a367b9b0e5d9d2087989e28e 100644 (file)
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -11,6 +11,8 @@
 #ifndef __ASM_S390_PROCESSOR_H
 #define __ASM_S390_PROCESSOR_H
 
+#ifndef __ASSEMBLY__
+
 #include <linux/linkage.h>
 #include <linux/irqflags.h>
 #include <asm/cpu.h>
@@ -348,23 +350,6 @@ extern void (*s390_base_ext_handler_fn)(void);
 
 #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL
 
-/*
- * Helper macro for exception table entries
- */
-#ifndef CONFIG_64BIT
-#define EX_TABLE(_fault,_target)                       \
-       ".section __ex_table,\"a\"\n"                   \
-       "       .align 4\n"                             \
-       "       .long  " #_fault "," #_target "\n"      \
-       ".previous\n"
-#else
-#define EX_TABLE(_fault,_target)                       \
-       ".section __ex_table,\"a\"\n"                   \
-       "       .align 8\n"                             \
-       "       .quad  " #_fault "," #_target "\n"      \
-       ".previous\n"
-#endif
-
 extern int memcpy_real(void *, void *, size_t);
 extern void memcpy_absolute(void *, void *, size_t);
 
@@ -375,4 +360,25 @@ extern void memcpy_absolute(void *, void *, size_t);
        memcpy_absolute(&(dest), &__tmp, sizeof(__tmp));        \
 }
 
-#endif                                 /* __ASM_S390_PROCESSOR_H           */
+/*
+ * Helper macro for exception table entries
+ */
+#define EX_TABLE(_fault, _target)      \
+       ".section __ex_table,\"a\"\n"   \
+       ".align 4\n"                    \
+       ".long  (" #_fault ") - .\n"    \
+       ".long  (" #_target ") - .\n"   \
+       ".previous\n"
+
+#else /* __ASSEMBLY__ */
+
+#define EX_TABLE(_fault, _target)      \
+       .section __ex_table,"a" ;       \
+       .align  4 ;                     \
+       .long   (_fault) - . ;          \
+       .long   (_target) - . ;         \
+       .previous
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* __ASM_S390_PROCESSOR_H */

diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index a8ab18b18b54d0c87e8c1c6f7a160352fb7d073e..34268df959a329dd71fd7e6fae51a308094f0624 100644 (file)
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -76,9 +76,22 @@ static inline int __range_ok(unsigned long addr, unsigned long size)
 
 struct exception_table_entry
 {
-        unsigned long insn, fixup;
+       int insn, fixup;
 };
 
+static inline unsigned long extable_insn(const struct exception_table_entry *x)
+{
+       return (unsigned long)&x->insn + x->insn;
+}
+
+static inline unsigned long extable_fixup(const struct exception_table_entry *x)
+{
+       return (unsigned long)&x->fixup + x->fixup;
+}
+
+#define ARCH_HAS_SORT_EXTABLE
+#define ARCH_HAS_SEARCH_EXTABLE
+
 struct uaccess_ops {
        size_t (*copy_from_user)(size_t, const void __user *, void *);
        size_t (*copy_from_user_small)(size_t, const void __user *, void *);

diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index f4bcdc01bfc81309c6302fae2f6eae023a7668e1..e8000d5ff533bce9e3a5fb801a595f05a7fce00c 100644 (file)
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -255,14 +255,14 @@ static __init void setup_topology(void)
 
 static void early_pgm_check_handler(void)
 {
-       unsigned long addr;
        const struct exception_table_entry *fixup;
+       unsigned long addr;
 
        addr = S390_lowcore.program_old_psw.addr;
        fixup = search_exception_tables(addr & PSW_ADDR_INSN);
        if (!fixup)
                disabled_wait(0);
-       S390_lowcore.program_old_psw.addr = fixup->fixup | PSW_ADDR_AMODE;
+       S390_lowcore.program_old_psw.addr = extable_fixup(fixup)|PSW_ADDR_AMODE;
 }
 
 static noinline __init void setup_lowcore_early(void)

diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index 95e9d93d4f41054d9aac5b03431d5441494addbf..7549985402f726515e49694708957e710c50e17e 100644 (file)
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -10,6 +10,7 @@
 
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <asm/processor.h>
 #include <asm/cache.h>
 #include <asm/errno.h>
 #include <asm/ptrace.h>
@@ -1008,9 +1009,7 @@ sie_fault:
 .Lhost_id:
        .quad   0
 
-       .section __ex_table,"a"
-       .quad   sie_loop,sie_fault
-       .previous
+       EX_TABLE(sie_loop,sie_fault)
 #endif
 
                .section .rodata, "a"

diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c
index 8aa634f5944b4be9f4f6a1d810de9a2d65e26f7b..d1c7214e157c2cdaab142541c4b7eec3420b5bb1 100644 (file)
--- a/arch/s390/kernel/kprobes.c
+++ b/arch/s390/kernel/kprobes.c
@@ -547,7 +547,7 @@ static int __kprobes kprobe_trap_handler(struct pt_regs *regs, int trapnr)
                 */
                entry = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
                if (entry) {
-                       regs->psw.addr = entry->fixup | PSW_ADDR_AMODE;
+                       regs->psw.addr = extable_fixup(entry) | PSW_ADDR_AMODE;
                        return 1;
                }
 

diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c
index 29af52628e8b1cd48859163cbc70d3a5f472074c..befc761fc154b2e96b1762600480ddb2b801d7c5 100644 (file)
--- a/arch/s390/kernel/traps.c
+++ b/arch/s390/kernel/traps.c
@@ -322,7 +322,7 @@ static void __kprobes do_trap(struct pt_regs *regs,
                 const struct exception_table_entry *fixup;
                 fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
                 if (fixup)
-                        regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
+                       regs->psw.addr = extable_fixup(fixup) | PSW_ADDR_AMODE;
                else {
                        enum bug_trap_type btt;
 

diff --git a/arch/s390/mm/Makefile b/arch/s390/mm/Makefile
index d98fe9004a524b8e4f11a4de5e099e56103beddf..0f5536b0c1a1c5368ff9976ff70beb873e92675b 100644 (file)
--- a/arch/s390/mm/Makefile
+++ b/arch/s390/mm/Makefile
@@ -3,7 +3,7 @@
 #
 
 obj-y   := init.o fault.o extmem.o mmap.o vmem.o pgtable.o maccess.o \
-           page-states.o gup.o
+           page-states.o gup.o extable.o
 obj-$(CONFIG_CMM) += cmm.o
 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
 obj-$(CONFIG_DEBUG_SET_MODULE_RONX) += pageattr.o

diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c
new file mode 100644 (file)
index 0000000..4d1ee88
--- /dev/null
+++ b/arch/s390/mm/extable.c
@@ -0,0 +1,81 @@
+#include <linux/module.h>
+#include <linux/sort.h>
+#include <asm/uaccess.h>
+
+/*
+ * Search one exception table for an entry corresponding to the
+ * given instruction address, and return the address of the entry,
+ * or NULL if none is found.
+ * We use a binary search, and thus we assume that the table is
+ * already sorted.
+ */
+const struct exception_table_entry *
+search_extable(const struct exception_table_entry *first,
+              const struct exception_table_entry *last,
+              unsigned long value)
+{
+       const struct exception_table_entry *mid;
+       unsigned long addr;
+
+       while (first <= last) {
+               mid = ((last - first) >> 1) + first;
+               addr = extable_insn(mid);
+               if (addr < value)
+                       first = mid + 1;
+               else if (addr > value)
+                       last = mid - 1;
+               else
+                       return mid;
+       }
+       return NULL;
+}
+
+/*
+ * The exception table needs to be sorted so that the binary
+ * search that we use to find entries in it works properly.
+ * This is used both for the kernel exception table and for
+ * the exception tables of modules that get loaded.
+ *
+ */
+static int cmp_ex(const void *a, const void *b)
+{
+       const struct exception_table_entry *x = a, *y = b;
+
+       /* This compare is only valid after normalization. */
+       return x->insn - y->insn;
+}
+
+void sort_extable(struct exception_table_entry *start,
+                 struct exception_table_entry *finish)
+{
+       struct exception_table_entry *p;
+       int i;
+
+       /* Normalize entries to being relative to the start of the section */
+       for (p = start, i = 0; p < finish; p++, i += 8)
+               p->insn += i;
+       sort(start, finish - start, sizeof(*start), cmp_ex, NULL);
+       /* Denormalize all entries */
+       for (p = start, i = 0; p < finish; p++, i += 8)
+               p->insn -= i;
+}
+
+#ifdef CONFIG_MODULES
+/*
+ * If the exception table is sorted, any referring to the module init
+ * will be at the beginning or the end.
+ */
+void trim_init_extable(struct module *m)
+{
+       /* Trim the beginning */
+       while (m->num_exentries &&
+              within_module_init(extable_insn(&m->extable[0]), m)) {
+               m->extable++;
+               m->num_exentries--;
+       }
+       /* Trim the end */
+       while (m->num_exentries &&
+              within_module_init(extable_insn(&m->extable[m->num_exentries-1]), m))
+               m->num_exentries--;
+}
+#endif /* CONFIG_MODULES */

diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index 8b2cac1ddbc556a1c4446bc40aa28f0addb3682c..ac9122ca11529217e3843483042306a7a09a74cb 100644 (file)
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -163,7 +163,7 @@ static noinline void do_no_context(struct pt_regs *regs)
        /* Are we prepared to handle this kernel fault?  */
        fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
        if (fixup) {
-               regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
+               regs->psw.addr = extable_fixup(fixup) | PSW_ADDR_AMODE;
                return;
        }
 

diff --git a/scripts/sortextable.c b/scripts/sortextable.c
index 6acf834491050237548163230a2b89e7e53b9b61..f19ddc47304c8f1e2a52a4acff95697fa9b3c902 100644 (file)
--- a/scripts/sortextable.c
+++ b/scripts/sortextable.c
@@ -161,7 +161,7 @@ typedef void (*table_sort_t)(char *, int);
 #define SORTEXTABLE_64
 #include "sortextable.h"
 
-static int compare_x86_table(const void *a, const void *b)
+static int compare_relative_table(const void *a, const void *b)
 {
        int32_t av = (int32_t)r(a);
        int32_t bv = (int32_t)r(b);
@@ -173,7 +173,7 @@ static int compare_x86_table(const void *a, const void *b)
        return 0;
 }
 
-static void sort_x86_table(char *extab_image, int image_size)
+static void sort_relative_table(char *extab_image, int image_size)
 {
        int i;
 
@@ -188,7 +188,7 @@ static void sort_x86_table(char *extab_image, int image_size)
                i += 4;
        }
 
-       qsort(extab_image, image_size / 8, 8, compare_x86_table);
+       qsort(extab_image, image_size / 8, 8, compare_relative_table);
 
        /* Now denormalize. */
        i = 0;
@@ -245,9 +245,9 @@ do_file(char const *const fname)
                break;
        case EM_386:
        case EM_X86_64:
-               custom_sort = sort_x86_table;
-               break;
        case EM_S390:
+               custom_sort = sort_relative_table;
+               break;
        case EM_MIPS:
                break;
        }  /* end switch */