UPSTREAM: arm64: Modify stack trace and dump for use with irq_stack

This patch allows unwind_frame() to traverse from interrupt stack to task
stack correctly. It requires data from a dummy stack frame, created
during irq_stack_entry(), added by a later patch.

A similar approach is taken to modify dump_backtrace(), which expects to
find struct pt_regs underneath any call to functions marked __exception.
When on an irq_stack, the struct pt_regs is stored on the old task stack,
the location of which is stored in the dummy stack frame.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: AKASHI Takahiro <takahiro.akashi@linaro.org>
[james.morse: merged two patches, reworked for per_cpu irq_stacks, and
 no alignment guarantees, added irq_stack definitions]
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Bug: 30369029
Patchset: per-cpu-irq-stack

(cherry picked from commit 132cd887b5c54758d04bf25c52fa48f45e843a30)
Signed-off-by: Jeff Vander Stoep <jeffv@google.com>
Change-Id: I60b29291620a71ab7b6564730299d29f41ceb199

diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h
index 8e8d30684392b1065b0c5d1f65e7715e4028331c..e2f3f135a3bc89ad8d05b3a7bea6b0e9eae70fdd 100644 (file)
--- a/arch/arm64/include/asm/irq.h
+++ b/arch/arm64/include/asm/irq.h
@@ -1,10 +1,32 @@
 #ifndef __ASM_IRQ_H
 #define __ASM_IRQ_H
 
+#define IRQ_STACK_SIZE                 THREAD_SIZE
+#define IRQ_STACK_START_SP             THREAD_START_SP
+
+#ifndef __ASSEMBLER__
+
+#include <linux/percpu.h>
+
 #include <asm-generic/irq.h>
+#include <asm/thread_info.h>
 
 struct pt_regs;
 
+DECLARE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
+
+/*
+ * The highest address on the stack, and the first to be used. Used to
+ * find the dummy-stack frame put down by el?_irq() in entry.S.
+ */
+#define IRQ_STACK_PTR(cpu) ((unsigned long)per_cpu(irq_stack, cpu) + IRQ_STACK_START_SP)
+
+/*
+ * The offset from irq_stack_ptr where entry.S will store the original
+ * stack pointer. Used by unwind_frame() and dump_backtrace().
+ */
+#define IRQ_STACK_TO_TASK_STACK(ptr) *((unsigned long *)(ptr - 0x10));
+
 extern void set_handle_irq(void (*handle_irq)(struct pt_regs *));
 
 static inline int nr_legacy_irqs(void)
@@ -12,4 +34,14 @@ static inline int nr_legacy_irqs(void)
        return 0;
 }
 
+static inline bool on_irq_stack(unsigned long sp, int cpu)
+{
+       /* variable names the same as kernel/stacktrace.c */
+       unsigned long low = (unsigned long)per_cpu(irq_stack, cpu);
+       unsigned long high = low + IRQ_STACK_START_SP;
+
+       return (low <= sp && sp <= high);
+}
+
+#endif /* !__ASSEMBLER__ */
 #endif

diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 9f17ec071ee0e8a8b1380133cf319a9ad1c80de5..1e3cef578e219e651340af2748833325c036c1fc 100644 (file)
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -30,6 +30,9 @@
 
 unsigned long irq_err_count;
 
+/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned */
+DEFINE_PER_CPU(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack) __aligned(16);
+
 int arch_show_interrupts(struct seq_file *p, int prec)
 {
        show_ipi_list(p, prec);

diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index ccb6078ed9f20fb55132deb48504df3a3134a784..b947eeffa5b29419dd4b23b7a5b5a146e8e2f071 100644 (file)
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -20,6 +20,7 @@
 #include <linux/sched.h>
 #include <linux/stacktrace.h>
 
+#include <asm/irq.h>
 #include <asm/stacktrace.h>
 
 /*
@@ -39,17 +40,41 @@ int notrace unwind_frame(struct stackframe *frame)
 {
        unsigned long high, low;
        unsigned long fp = frame->fp;
+       unsigned long irq_stack_ptr;
+
+       /*
+        * Use raw_smp_processor_id() to avoid false-positives from
+        * CONFIG_DEBUG_PREEMPT. get_wchan() calls unwind_frame() on sleeping
+        * task stacks, we can be pre-empted in this case, so
+        * {raw_,}smp_processor_id() may give us the wrong value. Sleeping
+        * tasks can't ever be on an interrupt stack, so regardless of cpu,
+        * the checks will always fail.
+        */
+       irq_stack_ptr = IRQ_STACK_PTR(raw_smp_processor_id());
 
        low  = frame->sp;
-       high = ALIGN(low, THREAD_SIZE);
+       /* irq stacks are not THREAD_SIZE aligned */
+       if (on_irq_stack(frame->sp, raw_smp_processor_id()))
+               high = irq_stack_ptr;
+       else
+               high = ALIGN(low, THREAD_SIZE) - 0x20;
 
-       if (fp < low || fp > high - 0x18 || fp & 0xf)
+       if (fp < low || fp > high || fp & 0xf)
                return -EINVAL;
 
        frame->sp = fp + 0x10;
        frame->fp = *(unsigned long *)(fp);
        frame->pc = *(unsigned long *)(fp + 8);
 
+       /*
+        * Check whether we are going to walk through from interrupt stack
+        * to task stack.
+        * If we reach the end of the stack - and its an interrupt stack,
+        * read the original task stack pointer from the dummy frame.
+        */
+       if (frame->sp == irq_stack_ptr)
+               frame->sp = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr);
+
        return 0;
 }
 

diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index e9b9b53643936a121e8c73db99373d7e7cab9b48..8a0084541f84a33be6f95ddb9a16fac6b8a708c1 100644 (file)
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -146,6 +146,7 @@ static void dump_instr(const char *lvl, struct pt_regs *regs)
 static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
 {
        struct stackframe frame;
+       unsigned long irq_stack_ptr = IRQ_STACK_PTR(smp_processor_id());
 
        pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
 
@@ -180,9 +181,20 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
                if (ret < 0)
                        break;
                stack = frame.sp;
-               if (in_exception_text(where))
+               if (in_exception_text(where)) {
+                       /*
+                        * If we switched to the irq_stack before calling this
+                        * exception handler, then the pt_regs will be on the
+                        * task stack. The easiest way to tell is if the large
+                        * pt_regs would overlap with the end of the irq_stack.
+                        */
+                       if (stack < irq_stack_ptr &&
+                           (stack + sizeof(struct pt_regs)) > irq_stack_ptr)
+                               stack = IRQ_STACK_TO_TASK_STACK(irq_stack_ptr);
+
                        dump_mem("", "Exception stack", stack,
                                 stack + sizeof(struct pt_regs), false);
+               }
        }
 }