x86: Save rbp in pt_regs on irq entry

From the x86_64 low level interrupt handlers, the frame pointer is
saved right after the partial pt_regs frame.

rbp is not supposed to be part of the irq partial saved registers,
but it only requires to extend the pt_regs frame by 8 bytes to
do so, plus a tiny stack offset fixup on irq exit.

This changes a bit the semantics or get_irq_entry() that is supposed
to provide only the value of caller saved registers and the cpu
saved frame. However it's a win for unwinders that can walk through
stack frames on top of get_irq_regs() snapshots.

A noticeable impact is that it makes perf events cpu-clock and
task-clock events based callchains working on x86_64.

Let's then save rbp into the irq pt_regs.

As a result with:

	perf record -e cpu-clock perf bench sched messaging
	perf report --stdio

Before:
    20.94%             perf  [kernel.kallsyms]        [k] lock_acquire
                       |
                       --- lock_acquire
                          |
                          |--44.01%-- __write_nocancel
                          |
                          |--43.18%-- __read
                          |
                          |--6.08%-- fork
                          |          create_worker
                          |
                          |--0.88%-- _dl_fixup
                          |
                          |--0.65%-- do_lookup_x
                          |
                          |--0.53%-- __GI___libc_read
                           --4.67%-- [...]

After:
    19.23%         perf  [kernel.kallsyms]    [k] __lock_acquire
                   |
                   --- __lock_acquire
                      |
                      |--97.74%-- lock_acquire
                      |          |
                      |          |--21.82%-- _raw_spin_lock
                      |          |          |
                      |          |          |--37.26%-- unix_stream_recvmsg
                      |          |          |          sock_aio_read
                      |          |          |          do_sync_read
                      |          |          |          vfs_read
                      |          |          |          sys_read
                      |          |          |          system_call
                      |          |          |          __read
                      |          |          |
                      |          |          |--24.09%-- unix_stream_sendmsg
                      |          |          |          sock_aio_write
                      |          |          |          do_sync_write
                      |          |          |          vfs_write
                      |          |          |          sys_write
                      |          |          |          system_call
                      |          |          |          __write_nocancel

v2: Fix cfi annotations.

Reported-by: Soeren Sandmann Pedersen <sandmann@redhat.com>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: H. Peter Anvin <hpa@zytor.com
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Stephane Eranian <eranian@google.com>
Cc: Jan Beulich <JBeulich@novell.com>

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e3ba417e869766bb258ef489aa389f4933abf2a3..d3b895f375d3df70d641654be9258b995c21f950 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -299,17 +299,21 @@ ENDPROC(native_usergs_sysret64)
 ENTRY(save_args)
        XCPT_FRAME
        cld
-       movq_cfi rdi, RDI+16-ARGOFFSET
-       movq_cfi rsi, RSI+16-ARGOFFSET
-       movq_cfi rdx, RDX+16-ARGOFFSET
-       movq_cfi rcx, RCX+16-ARGOFFSET
-       movq_cfi rax, RAX+16-ARGOFFSET
-       movq_cfi  r8,  R8+16-ARGOFFSET
-       movq_cfi  r9,  R9+16-ARGOFFSET
-       movq_cfi r10, R10+16-ARGOFFSET
-       movq_cfi r11, R11+16-ARGOFFSET
-
-       leaq -ARGOFFSET+16(%rsp),%rdi   /* arg1 for handler */
+       /*
+        * start from rbp in pt_regs and jump over
+        * return address.
+        */
+       movq_cfi rdi, RDI+8-RBP
+       movq_cfi rsi, RSI+8-RBP
+       movq_cfi rdx, RDX+8-RBP
+       movq_cfi rcx, RCX+8-RBP
+       movq_cfi rax, RAX+8-RBP
+       movq_cfi  r8,  R8+8-RBP
+       movq_cfi  r9,  R9+8-RBP
+       movq_cfi r10, R10+8-RBP
+       movq_cfi r11, R11+8-RBP
+
+       leaq -RBP+8(%rsp),%rdi  /* arg1 for handler */
        movq_cfi rbp, 8         /* push %rbp */
        leaq 8(%rsp), %rbp              /* mov %rsp, %ebp */
        testl $3, CS(%rdi)
@@ -782,8 +786,9 @@ END(interrupt)
 
 /* 0(%rsp): ~(interrupt number) */
        .macro interrupt func
-       subq $ORIG_RAX-ARGOFFSET+8, %rsp
-       CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8
+       /* reserve pt_regs for scratch regs and rbp */
+       subq $ORIG_RAX-RBP, %rsp
+       CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
        call save_args
        PARTIAL_FRAME 0
        call \func
@@ -808,9 +813,14 @@ ret_from_intr:
        TRACE_IRQS_OFF
        decl PER_CPU_VAR(irq_count)
        leaveq
+
        CFI_RESTORE             rbp
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
+
+       /* we did not save rbx, restore only from ARGOFFSET */
+       addq $8, %rsp
+       CFI_ADJUST_CFA_OFFSET   -8
 exit_intr:
        GET_THREAD_INFO(%rcx)
        testl $3,CS-ARGOFFSET(%rsp)