x86/ftrace: Add -mfentry support to x86_32 with DYNAMIC_FTRACE set

x86_64 has had fentry support for some time. I did not add support to x86_32
as I was unsure if it will be used much in the future. It is still very much
used, and there's issues with function graph tracing with gcc playing around
with the mcount frames, causing function graph to panic. The fentry code
does not have this issue, and is able to cope as there is no frame to mess
up.

Note, this only adds support for fentry when DYNAMIC_FTRACE is set. There's
really no reason to not have that set, because the performance of the
machine drops significantly when it's not enabled.

Keep !DYNAMIC_FTRACE around to test it off, as there's still some archs
that have FTRACE but not DYNAMIC_FTRACE.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Reviewed-by: Masami Hiramatsu <mhiramat@kernel.org>
Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com>
Reviewed-by: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20170323143446.052202377@goodmis.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index cc98d5a294eee25c56209d92e38bdb08f379780d..8c17146427ca1a0d02b8814388f2b14039d543b1 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -127,7 +127,7 @@ config X86
        select HAVE_EBPF_JIT                    if X86_64
        select HAVE_EFFICIENT_UNALIGNED_ACCESS
        select HAVE_EXIT_THREAD
-       select HAVE_FENTRY                      if X86_64
+       select HAVE_FENTRY                      if X86_64 || DYNAMIC_FTRACE
        select HAVE_FTRACE_MCOUNT_RECORD
        select HAVE_FUNCTION_GRAPH_TRACER
        select HAVE_FUNCTION_TRACER

diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index 93e26647c3f292e45c555af1c8cd684fa18ef3fb..80518ec5b882aa64c465399600197a5e62f94e43 100644 (file)
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -9,26 +9,68 @@
 #include <asm/ftrace.h>
 
 #ifdef CONFIG_FUNCTION_TRACER
+
+#ifdef CC_USING_FENTRY
+# define function_hook __fentry__
+EXPORT_SYMBOL(__fentry__)
+#else
+# define function_hook mcount
+EXPORT_SYMBOL(mcount)
+#endif
+
 #ifdef CONFIG_DYNAMIC_FTRACE
 
-ENTRY(mcount)
+/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
+#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
+# define USING_FRAME_POINTER
+#endif
+
+#ifdef USING_FRAME_POINTER
+# define MCOUNT_FRAME                  1       /* using frame = true  */
+#else
+# define MCOUNT_FRAME                  0       /* using frame = false */
+#endif
+
+ENTRY(function_hook)
        ret
-END(mcount)
+END(function_hook)
 
 ENTRY(ftrace_caller)
 
+#ifdef USING_FRAME_POINTER
+# ifdef CC_USING_FENTRY
+       /*
+        * Frame pointers are of ip followed by bp.
+        * Since fentry is an immediate jump, we are left with
+        * parent-ip, function-ip. We need to add a frame with
+        * parent-ip followed by ebp.
+        */
+       pushl   4(%esp)                         /* parent ip */
        pushl   %ebp
        movl    %esp, %ebp
-
+       pushl   2*4(%esp)                       /* function ip */
+# endif
+       /* For mcount, the function ip is directly above */
+       pushl   %ebp
+       movl    %esp, %ebp
+#endif
        pushl   %eax
        pushl   %ecx
        pushl   %edx
        pushl   $0                              /* Pass NULL as regs pointer */
-       movl    5*4(%esp), %eax
-       /* Copy original ebp into %edx */
+
+#ifdef USING_FRAME_POINTER
+       /* Load parent ebp into edx */
        movl    4*4(%esp), %edx
+#else
+       /* There's no frame pointer, load the appropriate stack addr instead */
+       lea     4*4(%esp), %edx
+#endif
+
+       movl    (MCOUNT_FRAME+4)*4(%esp), %eax  /* load the rip */
        /* Get the parent ip */
-       movl    0x4(%edx), %edx
+       movl    4(%edx), %edx                   /* edx has ebp */
+
        movl    function_trace_op, %ecx
        subl    $MCOUNT_INSN_SIZE, %eax
 
@@ -40,7 +82,14 @@ ftrace_call:
        popl    %edx
        popl    %ecx
        popl    %eax
+#ifdef USING_FRAME_POINTER
        popl    %ebp
+# ifdef CC_USING_FENTRY
+       addl    $4,%esp                         /* skip function ip */
+       popl    %ebp                            /* this is the orig bp */
+       addl    $4, %esp                        /* skip parent ip */
+# endif
+#endif
 .Lftrace_ret:
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 .globl ftrace_graph_call
@@ -81,6 +130,10 @@ ENTRY(ftrace_regs_caller)
        pushl   %edx
        pushl   %ecx
        pushl   %ebx
+#ifdef CC_USING_FENTRY
+       /* Load 4 off of the parent ip addr into ebp */
+       lea     14*4(%esp), %ebp
+#endif
 
        movl    12*4(%esp), %eax                /* Load ip (1st parameter) */
        subl    $MCOUNT_INSN_SIZE, %eax         /* Adjust ip */
@@ -119,7 +172,7 @@ GLOBAL(ftrace_regs_call)
        jmp     .Lftrace_ret
 #else /* ! CONFIG_DYNAMIC_FTRACE */
 
-ENTRY(mcount)
+ENTRY(function_hook)
        cmpl    $__PAGE_OFFSET, %esp
        jb      ftrace_stub                     /* Paging not enabled yet? */
 
@@ -151,9 +204,8 @@ ftrace_stub:
        popl    %ecx
        popl    %eax
        jmp     ftrace_stub
-END(mcount)
+END(function_hook)
 #endif /* CONFIG_DYNAMIC_FTRACE */
-EXPORT_SYMBOL(mcount)
 #endif /* CONFIG_FUNCTION_TRACER */
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -161,9 +213,15 @@ ENTRY(ftrace_graph_caller)
        pushl   %eax
        pushl   %ecx
        pushl   %edx
-       movl    0xc(%esp), %eax
+       movl    3*4(%esp), %eax
+       /* Even with frame pointers, fentry doesn't have one here */
+#ifdef CC_USING_FENTRY
+       lea     4*4(%esp), %edx
+       movl    $0, %ecx
+#else
        lea     0x4(%ebp), %edx
        movl    (%ebp), %ecx
+#endif
        subl    $MCOUNT_INSN_SIZE, %eax
        call    prepare_ftrace_return
        popl    %edx
@@ -176,7 +234,11 @@ END(ftrace_graph_caller)
 return_to_handler:
        pushl   %eax
        pushl   %edx
+#ifdef CC_USING_FENTRY
+       movl    $0, %eax
+#else
        movl    %ebp, %eax
+#endif
        call    ftrace_return_to_handler
        movl    %eax, %ecx
        popl    %edx