kprobes/x86: Boost probes when reentering

Integrate prepare_singlestep() into setup_singlestep() to boost
up reenter probes, if possible.

Signed-off-by: Masami Hiramatsu <mhiramat@redhat.com>
Cc: systemtap <systemtap@sources.redhat.com>
Cc: DLE <dle-develop@lists.sourceforge.net>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Jim Keniston <jkenisto@us.ibm.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Anders Kaseorg <andersk@ksplice.com>
Cc: Tim Abbott <tabbott@ksplice.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Jason Baron <jbaron@redhat.com>
Cc: Mathieu Desnoyers <compudj@krystal.dyndns.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
LKML-Reference: <20100225133423.6725.12071.stgit@localhost6.localdomain6>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index 15177cdfdd008fcc0e2265214b0d3d123b188157..c69bb65006f3daaef11e83c1f5f7d3eafb6b79f0 100644 (file)
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -406,18 +406,6 @@ static void __kprobes restore_btf(void)
                update_debugctlmsr(current->thread.debugctlmsr);
 }
 
-static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
-{
-       clear_btf();
-       regs->flags |= X86_EFLAGS_TF;
-       regs->flags &= ~X86_EFLAGS_IF;
-       /* single step inline if the instruction is an int3 */
-       if (p->opcode == BREAKPOINT_INSTRUCTION)
-               regs->ip = (unsigned long)p->addr;
-       else
-               regs->ip = (unsigned long)p->ainsn.insn;
-}
-
 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
                                      struct pt_regs *regs)
 {
@@ -430,19 +418,38 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
 }
 
 static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
-                                      struct kprobe_ctlblk *kcb)
+                                      struct kprobe_ctlblk *kcb, int reenter)
 {
 #if !defined(CONFIG_PREEMPT)
        if (p->ainsn.boostable == 1 && !p->post_handler) {
                /* Boost up -- we can execute copied instructions directly */
-               reset_current_kprobe();
+               if (!reenter)
+                       reset_current_kprobe();
+               /*
+                * Reentering boosted probe doesn't reset current_kprobe,
+                * nor set current_kprobe, because it doesn't use single
+                * stepping.
+                */
                regs->ip = (unsigned long)p->ainsn.insn;
                preempt_enable_no_resched();
                return;
        }
 #endif
-       prepare_singlestep(p, regs);
-       kcb->kprobe_status = KPROBE_HIT_SS;
+       if (reenter) {
+               save_previous_kprobe(kcb);
+               set_current_kprobe(p, regs, kcb);
+               kcb->kprobe_status = KPROBE_REENTER;
+       } else
+               kcb->kprobe_status = KPROBE_HIT_SS;
+       /* Prepare real single stepping */
+       clear_btf();
+       regs->flags |= X86_EFLAGS_TF;
+       regs->flags &= ~X86_EFLAGS_IF;
+       /* single step inline if the instruction is an int3 */
+       if (p->opcode == BREAKPOINT_INSTRUCTION)
+               regs->ip = (unsigned long)p->addr;
+       else
+               regs->ip = (unsigned long)p->ainsn.insn;
 }
 
 /*
@@ -456,11 +463,8 @@ static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
        switch (kcb->kprobe_status) {
        case KPROBE_HIT_SSDONE:
        case KPROBE_HIT_ACTIVE:
-               save_previous_kprobe(kcb);
-               set_current_kprobe(p, regs, kcb);
                kprobes_inc_nmissed_count(p);
-               prepare_singlestep(p, regs);
-               kcb->kprobe_status = KPROBE_REENTER;
+               setup_singlestep(p, regs, kcb, 1);
                break;
        case KPROBE_HIT_SS:
                /* A probe has been hit in the codepath leading up to, or just
@@ -535,13 +539,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
                         * more here.
                         */
                        if (!p->pre_handler || !p->pre_handler(p, regs))
-                               setup_singlestep(p, regs, kcb);
+                               setup_singlestep(p, regs, kcb, 0);
                        return 1;
                }
        } else if (kprobe_running()) {
                p = __get_cpu_var(current_kprobe);
                if (p->break_handler && p->break_handler(p, regs)) {
-                       setup_singlestep(p, regs, kcb);
+                       setup_singlestep(p, regs, kcb, 0);
                        return 1;
                }
        } /* else: not a kprobe fault; let the kernel handle it */