powerpc/kprobes: Add support for KPROBES_ON_FTRACE

Allow kprobes to be placed on ftrace _mcount() call sites. This optimization
avoids the use of a trap, by riding on ftrace infrastructure.

This depends on HAVE_DYNAMIC_FTRACE_WITH_REGS which depends on MPROFILE_KERNEL,
which is only currently enabled on powerpc64le with newer toolchains.

Based on the x86 code by Masami.

Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
index 40f44d041fb4764386fb6f90f12397bf805c7a7d..930430c6aef68d4b73296f5ef23fb8526fbf273b 100644 (file)
--- a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
+++ b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt
@@ -27,7 +27,7 @@
     |       nios2: | TODO |
     |    openrisc: | TODO |
     |      parisc: | TODO |
-    |     powerpc: | TODO |
+    |     powerpc: |  ok  |
     |        s390: | TODO |
     |       score: | TODO |
     |          sh: | TODO |

diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 97a8bc8a095ce4199ad2e4e0c88c2933fc3b6987..f1c43d78056c165fe6153487a97957f6d3b4207a 100644 (file)
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -142,6 +142,7 @@ config PPC
        select HAVE_IRQ_EXIT_ON_IRQ_STACK
        select HAVE_KERNEL_GZIP
        select HAVE_KPROBES
+       select HAVE_KPROBES_ON_FTRACE
        select HAVE_KRETPROBES
        select HAVE_LIVEPATCH                   if HAVE_DYNAMIC_FTRACE_WITH_REGS
        select HAVE_MEMBLOCK

diff --git a/arch/powerpc/include/asm/kprobes.h b/arch/powerpc/include/asm/kprobes.h
index a843884aafaf3924b5cf7a8c85a14e000667f935..a83821f33ea36f8c005dab8e80dfdfcfd26a7058 100644 (file)
--- a/arch/powerpc/include/asm/kprobes.h
+++ b/arch/powerpc/include/asm/kprobes.h
@@ -103,6 +103,16 @@ extern int kprobe_exceptions_notify(struct notifier_block *self,
 extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr);
 extern int kprobe_handler(struct pt_regs *regs);
 extern int kprobe_post_handler(struct pt_regs *regs);
+#ifdef CONFIG_KPROBES_ON_FTRACE
+extern int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                          struct kprobe_ctlblk *kcb);
+#else
+static inline int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                                 struct kprobe_ctlblk *kcb)
+{
+       return 0;
+}
+#endif
 #else
 static inline int kprobe_handler(struct pt_regs *regs) { return 0; }
 static inline int kprobe_post_handler(struct pt_regs *regs) { return 0; }

diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 811f441a125f684f91fa2b85dfe4dd8c685fb065..3e461637b64df2e556a942b9ca685d9b58889d38 100644 (file)
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -97,6 +97,7 @@ obj-$(CONFIG_BOOTX_TEXT)      += btext.o
 obj-$(CONFIG_SMP)              += smp.o
 obj-$(CONFIG_KPROBES)          += kprobes.o
 obj-$(CONFIG_OPTPROBES)                += optprobes.o optprobes_head.o
+obj-$(CONFIG_KPROBES_ON_FTRACE)        += kprobes-ftrace.o
 obj-$(CONFIG_UPROBES)          += uprobes.o
 obj-$(CONFIG_PPC_UDBG_16550)   += legacy_serial.o udbg_16550.o
 obj-$(CONFIG_STACKTRACE)       += stacktrace.o
@@ -150,6 +151,8 @@ GCOV_PROFILE_machine_kexec_32.o := n
 UBSAN_SANITIZE_machine_kexec_32.o := n
 GCOV_PROFILE_kprobes.o := n
 UBSAN_SANITIZE_kprobes.o := n
+GCOV_PROFILE_kprobes-ftrace.o := n
+UBSAN_SANITIZE_kprobes-ftrace.o := n
 UBSAN_SANITIZE_vdso.o := n
 
 extra-$(CONFIG_PPC_FPU)                += fpu.o

diff --git a/arch/powerpc/kernel/kprobes-ftrace.c b/arch/powerpc/kernel/kprobes-ftrace.c
new file mode 100644 (file)
index 0000000..6c089d9
--- /dev/null
+++ b/arch/powerpc/kernel/kprobes-ftrace.c
@@ -0,0 +1,104 @@
+/*
+ * Dynamic Ftrace based Kprobes Optimization
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) Hitachi Ltd., 2012
+ * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
+ *               IBM Corporation
+ */
+#include <linux/kprobes.h>
+#include <linux/ptrace.h>
+#include <linux/hardirq.h>
+#include <linux/preempt.h>
+#include <linux/ftrace.h>
+
+static nokprobe_inline
+int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                     struct kprobe_ctlblk *kcb, unsigned long orig_nip)
+{
+       /*
+        * Emulate singlestep (and also recover regs->nip)
+        * as if there is a nop
+        */
+       regs->nip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
+       if (unlikely(p->post_handler)) {
+               kcb->kprobe_status = KPROBE_HIT_SSDONE;
+               p->post_handler(p, regs, 0);
+       }
+       __this_cpu_write(current_kprobe, NULL);
+       if (orig_nip)
+               regs->nip = orig_nip;
+       return 1;
+}
+
+int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
+                   struct kprobe_ctlblk *kcb)
+{
+       if (kprobe_ftrace(p))
+               return __skip_singlestep(p, regs, kcb, 0);
+       else
+               return 0;
+}
+NOKPROBE_SYMBOL(skip_singlestep);
+
+/* Ftrace callback handler for kprobes */
+void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,
+                          struct ftrace_ops *ops, struct pt_regs *regs)
+{
+       struct kprobe *p;
+       struct kprobe_ctlblk *kcb;
+       unsigned long flags;
+
+       /* Disable irq for emulating a breakpoint and avoiding preempt */
+       local_irq_save(flags);
+       hard_irq_disable();
+
+       p = get_kprobe((kprobe_opcode_t *)nip);
+       if (unlikely(!p) || kprobe_disabled(p))
+               goto end;
+
+       kcb = get_kprobe_ctlblk();
+       if (kprobe_running()) {
+               kprobes_inc_nmissed_count(p);
+       } else {
+               unsigned long orig_nip = regs->nip;
+
+               /*
+                * On powerpc, NIP is *before* this instruction for the
+                * pre handler
+                */
+               regs->nip -= MCOUNT_INSN_SIZE;
+
+               __this_cpu_write(current_kprobe, p);
+               kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+               if (!p->pre_handler || !p->pre_handler(p, regs))
+                       __skip_singlestep(p, regs, kcb, orig_nip);
+               /*
+                * If pre_handler returns !0, it sets regs->nip and
+                * resets current kprobe.
+                */
+       }
+end:
+       local_irq_restore(flags);
+}
+NOKPROBE_SYMBOL(kprobe_ftrace_handler);
+
+int arch_prepare_kprobe_ftrace(struct kprobe *p)
+{
+       p->ainsn.insn = NULL;
+       p->ainsn.boostable = -1;
+       return 0;
+}

diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
index b71922618ed25f1d9e6fc8b257cb6554de84e2b1..50aabf819787b0b40122e63f1d5c0a2a30ccfea9 100644 (file)
--- a/arch/powerpc/kernel/kprobes.c
+++ b/arch/powerpc/kernel/kprobes.c
@@ -192,7 +192,11 @@ static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
 bool arch_function_offset_within_entry(unsigned long offset)
 {
 #ifdef PPC64_ELF_ABI_v2
+#ifdef CONFIG_KPROBES_ON_FTRACE
+       return offset <= 16;
+#else
        return offset <= 8;
+#endif
 #else
        return !offset;
 #endif
@@ -301,7 +305,9 @@ int __kprobes kprobe_handler(struct pt_regs *regs)
                        }
                        p = __this_cpu_read(current_kprobe);
                        if (p->break_handler && p->break_handler(p, regs)) {
-                               goto ss_probe;
+                               if (!skip_singlestep(p, regs, kcb))
+                                       goto ss_probe;
+                               ret = 1;
                        }
                }
                goto no_kprobe;