Commit | Line | Data |
---|---|---|
3d442233 JA |
1 | /* |
2 | * Generic helpers for smp ipi calls | |
3 | * | |
4 | * (C) Jens Axboe <jens.axboe@oracle.com> 2008 | |
5 | * | |
6 | */ | |
7 | #include <linux/init.h> | |
8 | #include <linux/module.h> | |
9 | #include <linux/percpu.h> | |
10 | #include <linux/rcupdate.h> | |
59190f42 | 11 | #include <linux/rculist.h> |
3d442233 | 12 | #include <linux/smp.h> |
8969a5ed | 13 | #include <linux/cpu.h> |
3d442233 JA |
14 | |
15 | static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); | |
8969a5ed PZ |
16 | |
17 | static struct { | |
18 | struct list_head queue; | |
19 | spinlock_t lock; | |
20 | } call_function __cacheline_aligned_in_smp = { | |
21 | .queue = LIST_HEAD_INIT(call_function.queue), | |
22 | .lock = __SPIN_LOCK_UNLOCKED(call_function.lock), | |
23 | }; | |
3d442233 JA |
24 | |
25 | enum { | |
26 | CSD_FLAG_WAIT = 0x01, | |
8969a5ed | 27 | CSD_FLAG_LOCK = 0x02, |
3d442233 JA |
28 | }; |
29 | ||
30 | struct call_function_data { | |
31 | struct call_single_data csd; | |
32 | spinlock_t lock; | |
33 | unsigned int refs; | |
8969a5ed | 34 | cpumask_var_t cpumask; |
3d442233 JA |
35 | }; |
36 | ||
37 | struct call_single_queue { | |
38 | struct list_head list; | |
39 | spinlock_t lock; | |
40 | }; | |
41 | ||
8969a5ed PZ |
42 | static DEFINE_PER_CPU(struct call_function_data, cfd_data) = { |
43 | .lock = __SPIN_LOCK_UNLOCKED(cfd_data.lock), | |
44 | }; | |
45 | ||
46 | static int | |
47 | hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu) | |
48 | { | |
49 | long cpu = (long)hcpu; | |
50 | struct call_function_data *cfd = &per_cpu(cfd_data, cpu); | |
51 | ||
52 | switch (action) { | |
53 | case CPU_UP_PREPARE: | |
54 | case CPU_UP_PREPARE_FROZEN: | |
55 | if (!alloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL, | |
56 | cpu_to_node(cpu))) | |
57 | return NOTIFY_BAD; | |
58 | break; | |
59 | ||
60 | #ifdef CONFIG_CPU_HOTPLUG | |
61 | case CPU_UP_CANCELED: | |
62 | case CPU_UP_CANCELED_FROZEN: | |
63 | ||
64 | case CPU_DEAD: | |
65 | case CPU_DEAD_FROZEN: | |
66 | free_cpumask_var(cfd->cpumask); | |
67 | break; | |
68 | #endif | |
69 | }; | |
70 | ||
71 | return NOTIFY_OK; | |
72 | } | |
73 | ||
74 | static struct notifier_block __cpuinitdata hotplug_cfd_notifier = { | |
75 | .notifier_call = hotplug_cfd, | |
76 | }; | |
77 | ||
7babe8db | 78 | static int __cpuinit init_call_single_data(void) |
3d442233 | 79 | { |
8969a5ed | 80 | void *cpu = (void *)(long)smp_processor_id(); |
3d442233 JA |
81 | int i; |
82 | ||
83 | for_each_possible_cpu(i) { | |
84 | struct call_single_queue *q = &per_cpu(call_single_queue, i); | |
85 | ||
86 | spin_lock_init(&q->lock); | |
87 | INIT_LIST_HEAD(&q->list); | |
88 | } | |
8969a5ed PZ |
89 | |
90 | hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu); | |
91 | register_cpu_notifier(&hotplug_cfd_notifier); | |
92 | ||
7babe8db | 93 | return 0; |
3d442233 | 94 | } |
7babe8db | 95 | early_initcall(init_call_single_data); |
3d442233 | 96 | |
8969a5ed PZ |
97 | /* |
98 | * csd_wait/csd_complete are used for synchronous ipi calls | |
99 | */ | |
100 | static void csd_wait_prepare(struct call_single_data *data) | |
3d442233 | 101 | { |
8969a5ed PZ |
102 | data->flags |= CSD_FLAG_WAIT; |
103 | } | |
104 | ||
105 | static void csd_complete(struct call_single_data *data) | |
106 | { | |
107 | if (data->flags & CSD_FLAG_WAIT) { | |
108 | /* | |
109 | * ensure we're all done before saying we are | |
110 | */ | |
111 | smp_mb(); | |
112 | data->flags &= ~CSD_FLAG_WAIT; | |
113 | } | |
114 | } | |
115 | ||
116 | static void csd_wait(struct call_single_data *data) | |
117 | { | |
118 | while (data->flags & CSD_FLAG_WAIT) | |
3d442233 | 119 | cpu_relax(); |
8969a5ed PZ |
120 | } |
121 | ||
122 | /* | |
123 | * csd_lock/csd_unlock used to serialize access to per-cpu csd resources | |
124 | * | |
125 | * For non-synchronous ipi calls the csd can still be in use by the previous | |
126 | * function call. For multi-cpu calls its even more interesting as we'll have | |
127 | * to ensure no other cpu is observing our csd. | |
128 | */ | |
129 | static void csd_lock(struct call_single_data *data) | |
130 | { | |
131 | while (data->flags & CSD_FLAG_LOCK) | |
132 | cpu_relax(); | |
133 | data->flags = CSD_FLAG_LOCK; | |
134 | ||
135 | /* | |
136 | * prevent CPU from reordering the above assignment to ->flags | |
137 | * with any subsequent assignments to other fields of the | |
138 | * specified call_single_data structure. | |
139 | */ | |
140 | ||
141 | smp_mb(); | |
142 | } | |
143 | ||
144 | static void csd_unlock(struct call_single_data *data) | |
145 | { | |
146 | WARN_ON(!(data->flags & CSD_FLAG_LOCK)); | |
147 | /* | |
148 | * ensure we're all done before releasing data | |
149 | */ | |
150 | smp_mb(); | |
151 | data->flags &= ~CSD_FLAG_LOCK; | |
3d442233 JA |
152 | } |
153 | ||
154 | /* | |
155 | * Insert a previously allocated call_single_data element for execution | |
156 | * on the given CPU. data must already have ->func, ->info, and ->flags set. | |
157 | */ | |
158 | static void generic_exec_single(int cpu, struct call_single_data *data) | |
159 | { | |
160 | struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); | |
161 | int wait = data->flags & CSD_FLAG_WAIT, ipi; | |
162 | unsigned long flags; | |
163 | ||
164 | spin_lock_irqsave(&dst->lock, flags); | |
165 | ipi = list_empty(&dst->list); | |
166 | list_add_tail(&data->list, &dst->list); | |
167 | spin_unlock_irqrestore(&dst->lock, flags); | |
168 | ||
561920a0 | 169 | /* |
15d0d3b3 NP |
170 | * The list addition should be visible before sending the IPI |
171 | * handler locks the list to pull the entry off it because of | |
172 | * normal cache coherency rules implied by spinlocks. | |
173 | * | |
174 | * If IPIs can go out of order to the cache coherency protocol | |
175 | * in an architecture, sufficient synchronisation should be added | |
176 | * to arch code to make it appear to obey cache coherency WRT | |
177 | * locking and barrier primitives. Generic code isn't really equipped | |
178 | * to do the right thing... | |
561920a0 | 179 | */ |
561920a0 | 180 | |
3d442233 JA |
181 | if (ipi) |
182 | arch_send_call_function_single_ipi(cpu); | |
183 | ||
184 | if (wait) | |
8969a5ed | 185 | csd_wait(data); |
3d442233 JA |
186 | } |
187 | ||
188 | /* | |
189 | * Invoked by arch to handle an IPI for call function. Must be called with | |
190 | * interrupts disabled. | |
191 | */ | |
192 | void generic_smp_call_function_interrupt(void) | |
193 | { | |
194 | struct call_function_data *data; | |
195 | int cpu = get_cpu(); | |
196 | ||
15d0d3b3 NP |
197 | /* |
198 | * Ensure entry is visible on call_function_queue after we have | |
199 | * entered the IPI. See comment in smp_call_function_many. | |
200 | * If we don't have this, then we may miss an entry on the list | |
201 | * and never get another IPI to process it. | |
202 | */ | |
203 | smp_mb(); | |
204 | ||
3d442233 JA |
205 | /* |
206 | * It's ok to use list_for_each_rcu() here even though we may delete | |
207 | * 'pos', since list_del_rcu() doesn't clear ->next | |
208 | */ | |
8969a5ed | 209 | list_for_each_entry_rcu(data, &call_function.queue, csd.list) { |
3d442233 JA |
210 | int refs; |
211 | ||
8969a5ed PZ |
212 | spin_lock(&data->lock); |
213 | if (!cpumask_test_cpu(cpu, data->cpumask)) { | |
214 | spin_unlock(&data->lock); | |
3d442233 | 215 | continue; |
8969a5ed PZ |
216 | } |
217 | cpumask_clear_cpu(cpu, data->cpumask); | |
218 | spin_unlock(&data->lock); | |
3d442233 JA |
219 | |
220 | data->csd.func(data->csd.info); | |
221 | ||
222 | spin_lock(&data->lock); | |
3d442233 | 223 | WARN_ON(data->refs == 0); |
8969a5ed PZ |
224 | refs = --data->refs; |
225 | if (!refs) { | |
226 | spin_lock(&call_function.lock); | |
227 | list_del_rcu(&data->csd.list); | |
228 | spin_unlock(&call_function.lock); | |
229 | } | |
3d442233 JA |
230 | spin_unlock(&data->lock); |
231 | ||
232 | if (refs) | |
233 | continue; | |
234 | ||
8969a5ed PZ |
235 | csd_complete(&data->csd); |
236 | csd_unlock(&data->csd); | |
3d442233 | 237 | } |
3d442233 JA |
238 | |
239 | put_cpu(); | |
240 | } | |
241 | ||
242 | /* | |
243 | * Invoked by arch to handle an IPI for call function single. Must be called | |
244 | * from the arch with interrupts disabled. | |
245 | */ | |
246 | void generic_smp_call_function_single_interrupt(void) | |
247 | { | |
248 | struct call_single_queue *q = &__get_cpu_var(call_single_queue); | |
249 | LIST_HEAD(list); | |
15d0d3b3 | 250 | unsigned int data_flags; |
3d442233 | 251 | |
15d0d3b3 NP |
252 | spin_lock(&q->lock); |
253 | list_replace_init(&q->list, &list); | |
254 | spin_unlock(&q->lock); | |
3d442233 | 255 | |
15d0d3b3 NP |
256 | while (!list_empty(&list)) { |
257 | struct call_single_data *data; | |
3d442233 | 258 | |
15d0d3b3 NP |
259 | data = list_entry(list.next, struct call_single_data, |
260 | list); | |
261 | list_del(&data->list); | |
3d442233 | 262 | |
3d442233 | 263 | /* |
15d0d3b3 NP |
264 | * 'data' can be invalid after this call if |
265 | * flags == 0 (when called through | |
266 | * generic_exec_single(), so save them away before | |
267 | * making the call. | |
3d442233 | 268 | */ |
15d0d3b3 NP |
269 | data_flags = data->flags; |
270 | ||
271 | data->func(data->info); | |
272 | ||
8969a5ed PZ |
273 | if (data_flags & CSD_FLAG_WAIT) |
274 | csd_complete(data); | |
275 | ||
276 | /* | |
277 | * Unlocked CSDs are valid through generic_exec_single() | |
278 | */ | |
279 | if (data_flags & CSD_FLAG_LOCK) | |
280 | csd_unlock(data); | |
3d442233 JA |
281 | } |
282 | } | |
283 | ||
d7240b98 SR |
284 | static DEFINE_PER_CPU(struct call_single_data, csd_data); |
285 | ||
3d442233 JA |
286 | /* |
287 | * smp_call_function_single - Run a function on a specific CPU | |
288 | * @func: The function to run. This must be fast and non-blocking. | |
289 | * @info: An arbitrary pointer to pass to the function. | |
3d442233 JA |
290 | * @wait: If true, wait until function has completed on other CPUs. |
291 | * | |
292 | * Returns 0 on success, else a negative status code. Note that @wait | |
293 | * will be implicitly turned on in case of allocation failures, since | |
294 | * we fall back to on-stack allocation. | |
295 | */ | |
296 | int smp_call_function_single(int cpu, void (*func) (void *info), void *info, | |
8691e5a8 | 297 | int wait) |
3d442233 | 298 | { |
8969a5ed PZ |
299 | struct call_single_data d = { |
300 | .flags = 0, | |
301 | }; | |
3d442233 | 302 | unsigned long flags; |
f73be6de PA |
303 | /* prevent preemption and reschedule on another processor, |
304 | as well as CPU removal */ | |
3d442233 | 305 | int me = get_cpu(); |
f73be6de | 306 | int err = 0; |
3d442233 JA |
307 | |
308 | /* Can deadlock when called with interrupts disabled */ | |
309 | WARN_ON(irqs_disabled()); | |
310 | ||
311 | if (cpu == me) { | |
312 | local_irq_save(flags); | |
313 | func(info); | |
314 | local_irq_restore(flags); | |
4f4b6c1a | 315 | } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { |
d7240b98 | 316 | struct call_single_data *data; |
3d442233 JA |
317 | |
318 | if (!wait) { | |
d7240b98 SR |
319 | /* |
320 | * We are calling a function on a single CPU | |
321 | * and we are not going to wait for it to finish. | |
8969a5ed PZ |
322 | * We use a per cpu data to pass the information to |
323 | * that CPU. Since all callers of this code will | |
324 | * use the same data, we must synchronize the | |
325 | * callers to prevent a new caller from corrupting | |
326 | * the data before the callee can access it. | |
d7240b98 SR |
327 | * |
328 | * The CSD_FLAG_LOCK is used to let us know when | |
329 | * the IPI handler is done with the data. | |
330 | * The first caller will set it, and the callee | |
331 | * will clear it. The next caller must wait for | |
332 | * it to clear before we set it again. This | |
333 | * will make sure the callee is done with the | |
334 | * data before a new caller will use it. | |
335 | */ | |
8969a5ed PZ |
336 | data = &__get_cpu_var(csd_data); |
337 | csd_lock(data); | |
d7240b98 | 338 | } else { |
3d442233 | 339 | data = &d; |
8969a5ed | 340 | csd_wait_prepare(data); |
3d442233 JA |
341 | } |
342 | ||
343 | data->func = func; | |
344 | data->info = info; | |
345 | generic_exec_single(cpu, data); | |
f73be6de PA |
346 | } else { |
347 | err = -ENXIO; /* CPU not online */ | |
3d442233 JA |
348 | } |
349 | ||
350 | put_cpu(); | |
f73be6de | 351 | return err; |
3d442233 JA |
352 | } |
353 | EXPORT_SYMBOL(smp_call_function_single); | |
354 | ||
355 | /** | |
356 | * __smp_call_function_single(): Run a function on another CPU | |
357 | * @cpu: The CPU to run on. | |
358 | * @data: Pre-allocated and setup data structure | |
359 | * | |
360 | * Like smp_call_function_single(), but allow caller to pass in a pre-allocated | |
361 | * data structure. Useful for embedding @data inside other structures, for | |
362 | * instance. | |
363 | * | |
364 | */ | |
365 | void __smp_call_function_single(int cpu, struct call_single_data *data) | |
366 | { | |
367 | /* Can deadlock when called with interrupts disabled */ | |
368 | WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled()); | |
369 | ||
370 | generic_exec_single(cpu, data); | |
371 | } | |
372 | ||
ce47d974 RR |
373 | /* FIXME: Shim for archs using old arch_send_call_function_ipi API. */ |
374 | #ifndef arch_send_call_function_ipi_mask | |
375 | #define arch_send_call_function_ipi_mask(maskp) \ | |
376 | arch_send_call_function_ipi(*(maskp)) | |
377 | #endif | |
378 | ||
3d442233 | 379 | /** |
54b11e6d RR |
380 | * smp_call_function_many(): Run a function on a set of other CPUs. |
381 | * @mask: The set of cpus to run on (only runs on online subset). | |
3d442233 JA |
382 | * @func: The function to run. This must be fast and non-blocking. |
383 | * @info: An arbitrary pointer to pass to the function. | |
384 | * @wait: If true, wait (atomically) until function has completed on other CPUs. | |
385 | * | |
3d442233 JA |
386 | * If @wait is true, then returns once @func has returned. Note that @wait |
387 | * will be implicitly turned on in case of allocation failures, since | |
388 | * we fall back to on-stack allocation. | |
389 | * | |
390 | * You must not call this function with disabled interrupts or from a | |
391 | * hardware interrupt handler or from a bottom half handler. Preemption | |
392 | * must be disabled when calling this function. | |
393 | */ | |
54b11e6d RR |
394 | void smp_call_function_many(const struct cpumask *mask, |
395 | void (*func)(void *), void *info, | |
396 | bool wait) | |
3d442233 | 397 | { |
54b11e6d | 398 | struct call_function_data *data; |
3d442233 | 399 | unsigned long flags; |
8969a5ed | 400 | int cpu, next_cpu, me = smp_processor_id(); |
3d442233 JA |
401 | |
402 | /* Can deadlock when called with interrupts disabled */ | |
403 | WARN_ON(irqs_disabled()); | |
404 | ||
54b11e6d RR |
405 | /* So, what's a CPU they want? Ignoring this one. */ |
406 | cpu = cpumask_first_and(mask, cpu_online_mask); | |
8969a5ed | 407 | if (cpu == me) |
54b11e6d RR |
408 | cpu = cpumask_next_and(cpu, mask, cpu_online_mask); |
409 | /* No online cpus? We're done. */ | |
410 | if (cpu >= nr_cpu_ids) | |
411 | return; | |
412 | ||
413 | /* Do we have another CPU which isn't us? */ | |
414 | next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask); | |
8969a5ed | 415 | if (next_cpu == me) |
54b11e6d RR |
416 | next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask); |
417 | ||
418 | /* Fastpath: do that cpu by itself. */ | |
419 | if (next_cpu >= nr_cpu_ids) { | |
420 | smp_call_function_single(cpu, func, info, wait); | |
421 | return; | |
3d442233 JA |
422 | } |
423 | ||
8969a5ed PZ |
424 | data = &__get_cpu_var(cfd_data); |
425 | csd_lock(&data->csd); | |
3d442233 | 426 | |
8969a5ed | 427 | spin_lock_irqsave(&data->lock, flags); |
54b11e6d | 428 | if (wait) |
8969a5ed PZ |
429 | csd_wait_prepare(&data->csd); |
430 | ||
3d442233 JA |
431 | data->csd.func = func; |
432 | data->csd.info = info; | |
8969a5ed PZ |
433 | cpumask_and(data->cpumask, mask, cpu_online_mask); |
434 | cpumask_clear_cpu(me, data->cpumask); | |
435 | data->refs = cpumask_weight(data->cpumask); | |
3d442233 | 436 | |
8969a5ed PZ |
437 | spin_lock(&call_function.lock); |
438 | /* | |
439 | * Place entry at the _HEAD_ of the list, so that any cpu still | |
440 | * observing the entry in generic_smp_call_function_interrupt() will | |
441 | * not miss any other list entries. | |
442 | */ | |
443 | list_add_rcu(&data->csd.list, &call_function.queue); | |
444 | spin_unlock(&call_function.lock); | |
445 | spin_unlock_irqrestore(&data->lock, flags); | |
3d442233 | 446 | |
561920a0 SS |
447 | /* |
448 | * Make the list addition visible before sending the ipi. | |
15d0d3b3 NP |
449 | * (IPIs must obey or appear to obey normal Linux cache coherency |
450 | * rules -- see comment in generic_exec_single). | |
561920a0 SS |
451 | */ |
452 | smp_mb(); | |
453 | ||
3d442233 | 454 | /* Send a message to all CPUs in the map */ |
8969a5ed | 455 | arch_send_call_function_ipi_mask(data->cpumask); |
3d442233 JA |
456 | |
457 | /* optionally wait for the CPUs to complete */ | |
54b11e6d | 458 | if (wait) |
8969a5ed | 459 | csd_wait(&data->csd); |
3d442233 | 460 | } |
54b11e6d | 461 | EXPORT_SYMBOL(smp_call_function_many); |
3d442233 JA |
462 | |
463 | /** | |
464 | * smp_call_function(): Run a function on all other CPUs. | |
465 | * @func: The function to run. This must be fast and non-blocking. | |
466 | * @info: An arbitrary pointer to pass to the function. | |
3d442233 JA |
467 | * @wait: If true, wait (atomically) until function has completed on other CPUs. |
468 | * | |
54b11e6d | 469 | * Returns 0. |
3d442233 JA |
470 | * |
471 | * If @wait is true, then returns once @func has returned; otherwise | |
472 | * it returns just before the target cpu calls @func. In case of allocation | |
473 | * failure, @wait will be implicitly turned on. | |
474 | * | |
475 | * You must not call this function with disabled interrupts or from a | |
476 | * hardware interrupt handler or from a bottom half handler. | |
477 | */ | |
8691e5a8 | 478 | int smp_call_function(void (*func)(void *), void *info, int wait) |
3d442233 | 479 | { |
3d442233 | 480 | preempt_disable(); |
54b11e6d | 481 | smp_call_function_many(cpu_online_mask, func, info, wait); |
3d442233 | 482 | preempt_enable(); |
54b11e6d | 483 | return 0; |
3d442233 JA |
484 | } |
485 | EXPORT_SYMBOL(smp_call_function); | |
486 | ||
487 | void ipi_call_lock(void) | |
488 | { | |
8969a5ed | 489 | spin_lock(&call_function.lock); |
3d442233 JA |
490 | } |
491 | ||
492 | void ipi_call_unlock(void) | |
493 | { | |
8969a5ed | 494 | spin_unlock(&call_function.lock); |
3d442233 JA |
495 | } |
496 | ||
497 | void ipi_call_lock_irq(void) | |
498 | { | |
8969a5ed | 499 | spin_lock_irq(&call_function.lock); |
3d442233 JA |
500 | } |
501 | ||
502 | void ipi_call_unlock_irq(void) | |
503 | { | |
8969a5ed | 504 | spin_unlock_irq(&call_function.lock); |
3d442233 | 505 | } |