ftrace: Add return address pointer to ftrace_ret_stack

Storing this value will help prevent unwinders from getting out of sync
with the function graph tracer ret_stack.  Now instead of needing a
stateful iterator, they can compare the return address pointer to find
the right ret_stack entry.

Note that an array of 50 ftrace_ret_stack structs is allocated for every
task.  So when an arch implements this, it will add either 200 or 400
bytes of memory usage per task (depending on whether it's a 32-bit or
64-bit platform).

Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Byungchul Park <byungchul.park@lge.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Nilay Vaish <nilayvaish@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/a95cfcc39e8f26b89a430c56926af0bb217bc0a1.1471607358.git.jpoimboe@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/Documentation/trace/ftrace-design.txt b/Documentation/trace/ftrace-design.txt
index dd5f916b351d1cdd7f11162cb497530973c53940..a273dd0bbaaa52a86691c84ae64e787a30d0c9b0 100644 (file)
--- a/Documentation/trace/ftrace-design.txt
+++ b/Documentation/trace/ftrace-design.txt
@@ -203,6 +203,17 @@ along to ftrace_push_return_trace() instead of a stub value of 0.
 
 Similarly, when you call ftrace_return_to_handler(), pass it the frame pointer.
 
+HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
+--------------------------------
+
+An arch may pass in a pointer to the return address on the stack.  This
+prevents potential stack unwinding issues where the unwinder gets out of
+sync with ret_stack and the wrong addresses are reported by
+ftrace_graph_ret_addr().
+
+Adding support for it is easy: just define the macro in asm/ftrace.h and
+pass the return address pointer as the 'retp' argument to
+ftrace_push_return_trace().
 
 HAVE_FTRACE_NMI_ENTER
 ---------------------

diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c
index 709ee1d6d4df4813c82ad1bb23dac7868e00f999..3f1759411d51bec32175992a2332ab07ca598a8a 100644 (file)
--- a/arch/arm/kernel/ftrace.c
+++ b/arch/arm/kernel/ftrace.c
@@ -218,7 +218,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
        }
 
        err = ftrace_push_return_trace(old, self_addr, &trace.depth,
-                                      frame_pointer);
+                                      frame_pointer, NULL);
        if (err == -EBUSY) {
                *parent = old;
                return;

diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index ebecf9aa33d12da8a564ea0314f59a71b89e64a0..40ad08ac569af76b1f58fa775101ef264c480039 100644 (file)
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -138,7 +138,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
                return;
 
        err = ftrace_push_return_trace(old, self_addr, &trace.depth,
-                                      frame_pointer);
+                                      frame_pointer, NULL);
        if (err == -EBUSY)
                return;
        else

diff --git a/arch/blackfin/kernel/ftrace.c b/arch/blackfin/kernel/ftrace.c
index 095de0fa044d0eae0a698e8f02fae14329b4cfb0..8dad7589b8436a7a63acacd36bcc51f9b94d91d4 100644 (file)
--- a/arch/blackfin/kernel/ftrace.c
+++ b/arch/blackfin/kernel/ftrace.c
@@ -107,7 +107,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
                return;
 
        if (ftrace_push_return_trace(*parent, self_addr, &trace.depth,
-                                    frame_pointer) == -EBUSY)
+                                    frame_pointer, NULL) == -EBUSY)
                return;
 
        trace.func = self_addr;

diff --git a/arch/microblaze/kernel/ftrace.c b/arch/microblaze/kernel/ftrace.c
index fc7b48a52cd554c74be87e2185f7594c68c20beb..d57563c58a26be43672098d9895d9107c945e8d8 100644 (file)
--- a/arch/microblaze/kernel/ftrace.c
+++ b/arch/microblaze/kernel/ftrace.c
@@ -63,7 +63,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
                return;
        }
 
-       err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0);
+       err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0, NULL);
        if (err == -EBUSY) {
                *parent = old;
                return;

diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c
index 937c54bc8ccc499504c561f39c026b2c58417b50..30a3b75e88eb6a3310808d8c743739efa3f98fb7 100644 (file)
--- a/arch/mips/kernel/ftrace.c
+++ b/arch/mips/kernel/ftrace.c
@@ -382,8 +382,8 @@ void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra,
        if (unlikely(faulted))
                goto out;
 
-       if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp)
-           == -EBUSY) {
+       if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp,
+                                    NULL) == -EBUSY) {
                *parent_ra_addr = old_parent_ra;
                return;
        }

diff --git a/arch/parisc/kernel/ftrace.c b/arch/parisc/kernel/ftrace.c
index a828a0adf52c0b19d14eb12219bf718c3cf37704..5a5506a35395c411f91a61a59ebd50ed3ced38b6 100644 (file)
--- a/arch/parisc/kernel/ftrace.c
+++ b/arch/parisc/kernel/ftrace.c
@@ -48,7 +48,7 @@ static void __hot prepare_ftrace_return(unsigned long *parent,
                return;
 
         if (ftrace_push_return_trace(old, self_addr, &trace.depth,
-                       0 ) == -EBUSY)
+                                    0, NULL) == -EBUSY)
                 return;
 
        /* activate parisc_return_to_handler() as return point */

diff --git a/arch/powerpc/kernel/ftrace.c b/arch/powerpc/kernel/ftrace.c
index cc52d9795f88d5d6989f89adbf876030429402f1..a95639b8d4ac5d72a7be17d2b1e20e3f5fa874a1 100644 (file)
--- a/arch/powerpc/kernel/ftrace.c
+++ b/arch/powerpc/kernel/ftrace.c
@@ -593,7 +593,8 @@ unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
        if (!ftrace_graph_entry(&trace))
                goto out;
 
-       if (ftrace_push_return_trace(parent, ip, &trace.depth, 0) == -EBUSY)
+       if (ftrace_push_return_trace(parent, ip, &trace.depth, 0,
+                                    NULL) == -EBUSY)
                goto out;
 
        parent = return_hooker;

diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c
index 0f7bfeba6da6a3632f2705ebbaccc065ffd356a2..60a8a4e207edbc080a15b809b3efd97ea245942a 100644 (file)
--- a/arch/s390/kernel/ftrace.c
+++ b/arch/s390/kernel/ftrace.c
@@ -209,7 +209,8 @@ unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
        /* Only trace if the calling function expects to. */
        if (!ftrace_graph_entry(&trace))
                goto out;
-       if (ftrace_push_return_trace(parent, ip, &trace.depth, 0) == -EBUSY)
+       if (ftrace_push_return_trace(parent, ip, &trace.depth, 0,
+                                    NULL) == -EBUSY)
                goto out;
        parent = (unsigned long) return_to_handler;
 out:

diff --git a/arch/sh/kernel/ftrace.c b/arch/sh/kernel/ftrace.c
index 38993e09ef03ef371932707058c57bd15dd2d875..95eccd49672f40ca7f5e10f998c3226581221c29 100644 (file)
--- a/arch/sh/kernel/ftrace.c
+++ b/arch/sh/kernel/ftrace.c
@@ -382,7 +382,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
                return;
        }
 
-       err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0);
+       err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0, NULL);
        if (err == -EBUSY) {
                __raw_writel(old, parent);
                return;

diff --git a/arch/sparc/kernel/ftrace.c b/arch/sparc/kernel/ftrace.c
index 0a2d2ddff543fd325709fc46eaf48d438d06b88b..6bcff698069bf1fa57c1d84df180c1cb67f18a65 100644 (file)
--- a/arch/sparc/kernel/ftrace.c
+++ b/arch/sparc/kernel/ftrace.c
@@ -131,7 +131,7 @@ unsigned long prepare_ftrace_return(unsigned long parent,
                return parent + 8UL;
 
        if (ftrace_push_return_trace(parent, self_addr, &trace.depth,
-                                    frame_pointer) == -EBUSY)
+                                    frame_pointer, NULL) == -EBUSY)
                return parent + 8UL;
 
        trace.func = self_addr;

diff --git a/arch/tile/kernel/ftrace.c b/arch/tile/kernel/ftrace.c
index 4a572088b270bec26a0b10cce0453ab2d15e6c20..b827a418b155021104d27de2d821cf81e59babce 100644 (file)
--- a/arch/tile/kernel/ftrace.c
+++ b/arch/tile/kernel/ftrace.c
@@ -184,7 +184,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
        *parent = return_hooker;
 
        err = ftrace_push_return_trace(old, self_addr, &trace.depth,
-                                      frame_pointer);
+                                      frame_pointer, NULL);
        if (err == -EBUSY) {
                *parent = old;
                return;

diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index d036cfb4495db30015db71eee64cf7f633603a2b..ae3b1fb2f58261d643a8537ae81f9e93a4a58e4c 100644 (file)
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -1029,7 +1029,7 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
        }
 
        if (ftrace_push_return_trace(old, self_addr, &trace.depth,
-                   frame_pointer) == -EBUSY) {
+                                    frame_pointer, NULL) == -EBUSY) {
                *parent = old;
                return;
        }

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 4ad9ccc60e387381368874b34d810a688f8c740b..483e02a50d372d31dd7b2556d2b05fd8b4d32561 100644 (file)
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -798,6 +798,9 @@ struct ftrace_ret_stack {
 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
        unsigned long fp;
 #endif
+#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
+       unsigned long *retp;
+#endif
 };
 
 /*
@@ -809,7 +812,7 @@ extern void return_to_handler(void);
 
 extern int
 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
-                        unsigned long frame_pointer);
+                        unsigned long frame_pointer, unsigned long *retp);
 
 /*
  * Sometimes we don't want to trace a function with the function

diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 0e03ed0eac68870842359ccb975e0772f6b28456..f7212ec643e2dd4efdaffcc278e310135193d4e1 100644 (file)
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -119,7 +119,7 @@ print_graph_duration(struct trace_array *tr, unsigned long long duration,
 /* Add a function return address to the trace stack on thread info.*/
 int
 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
-                        unsigned long frame_pointer)
+                        unsigned long frame_pointer, unsigned long *retp)
 {
        unsigned long long calltime;
        int index;
@@ -173,6 +173,9 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
        current->ret_stack[index].subtime = 0;
 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
        current->ret_stack[index].fp = frame_pointer;
+#endif
+#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
+       current->ret_stack[index].retp = retp;
 #endif
        *depth = current->curr_ret_stack;