irq_work: Implement remote queueing

irq work currently only supports local callbacks. However its code
is mostly ready to run remote callbacks and we have some potential user.

The full nohz subsystem currently open codes its own remote irq work
on top of the scheduler ipi when it wants a CPU to reevaluate its next
tick. However this ad hoc solution bloats the scheduler IPI.

Lets just extend the irq work subsystem to support remote queuing on top
of the generic SMP IPI to handle this kind of user. This shouldn't add
noticeable overhead.

Suggested-by: Peter Zijlstra <peterz@infradead.org>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Kevin Hilman <khilman@linaro.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>

diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 19ae05d4b8ec26b6dc6fa5f568d2b25b698e3417..bf9422c3aefe22ddd7efa63b3f88502c0659b0b5 100644 (file)
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -33,6 +33,11 @@ void init_irq_work(struct irq_work *work, void (*func)(struct irq_work *))
 #define DEFINE_IRQ_WORK(name, _f) struct irq_work name = { .func = (_f), }
 
 bool irq_work_queue(struct irq_work *work);
+
+#ifdef CONFIG_SMP
+bool irq_work_queue_on(struct irq_work *work, int cpu);
+#endif
+
 void irq_work_run(void);
 void irq_work_sync(struct irq_work *work);
 

diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index 126f254614bfbe21690080ba75e51459d800cde8..4b0a890a304a2475c9e45bb171e41ca35f686ab5 100644 (file)
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -16,6 +16,7 @@
 #include <linux/tick.h>
 #include <linux/cpu.h>
 #include <linux/notifier.h>
+#include <linux/smp.h>
 #include <asm/processor.h>
 
 
@@ -55,12 +56,34 @@ void __weak arch_irq_work_raise(void)
         */
 }
 
+#ifdef CONFIG_SMP
 /*
- * Enqueue the irq_work @entry unless it's already pending
+ * Enqueue the irq_work @work on @cpu unless it's already pending
  * somewhere.
  *
  * Can be re-enqueued while the callback is still in progress.
  */
+bool irq_work_queue_on(struct irq_work *work, int cpu)
+{
+       /* All work should have been flushed before going offline */
+       WARN_ON_ONCE(cpu_is_offline(cpu));
+
+       /* Arch remote IPI send/receive backend aren't NMI safe */
+       WARN_ON_ONCE(in_nmi());
+
+       /* Only queue if not already pending */
+       if (!irq_work_claim(work))
+               return false;
+
+       if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
+               arch_send_call_function_single_ipi(cpu);
+
+       return true;
+}
+EXPORT_SYMBOL_GPL(irq_work_queue_on);
+#endif
+
+/* Enqueue the irq work @work on the current CPU */
 bool irq_work_queue(struct irq_work *work)
 {
        /* Only queue if not already pending */

diff --git a/kernel/smp.c b/kernel/smp.c
index 306f8180b0d53165c960d7844295d8c6a4d6151a..a1812d184aed05142808ce03126b84306bebddfb 100644 (file)
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -3,6 +3,7 @@
  *
  * (C) Jens Axboe <jens.axboe@oracle.com> 2008
  */
+#include <linux/irq_work.h>
 #include <linux/rcupdate.h>
 #include <linux/rculist.h>
 #include <linux/kernel.h>
@@ -210,6 +211,14 @@ void generic_smp_call_function_single_interrupt(void)
                csd->func(csd->info);
                csd_unlock(csd);
        }
+
+       /*
+        * Handle irq works queued remotely by irq_work_queue_on().
+        * Smp functions above are typically synchronous so they
+        * better run first since some other CPUs may be busy waiting
+        * for them.
+        */
+       irq_work_run();
 }
 
 /*