2 * (C) 2001, 2002, 2003, 2004 Rusty Russell
4 * This code is licenced under the GPL.
6 #include <linux/proc_fs.h>
8 #include <linux/init.h>
9 #include <linux/notifier.h>
10 #include <linux/sched/signal.h>
11 #include <linux/sched/hotplug.h>
12 #include <linux/sched/task.h>
13 #include <linux/unistd.h>
14 #include <linux/cpu.h>
15 #include <linux/oom.h>
16 #include <linux/rcupdate.h>
17 #include <linux/export.h>
18 #include <linux/bug.h>
19 #include <linux/kthread.h>
20 #include <linux/stop_machine.h>
21 #include <linux/mutex.h>
22 #include <linux/gfp.h>
23 #include <linux/suspend.h>
24 #include <linux/lockdep.h>
25 #include <linux/tick.h>
26 #include <linux/irq.h>
27 #include <linux/nmi.h>
28 #include <linux/smpboot.h>
29 #include <linux/relay.h>
30 #include <linux/slab.h>
31 #include <linux/percpu-rwsem.h>
33 #include <trace/events/power.h>
34 #define CREATE_TRACE_POINTS
35 #include <trace/events/cpuhp.h>
40 * cpuhp_cpu_state - Per cpu hotplug state storage
41 * @state: The current cpu state
42 * @target: The target state
43 * @thread: Pointer to the hotplug thread
44 * @should_run: Thread should execute
45 * @rollback: Perform a rollback
46 * @single: Single callback invocation
47 * @bringup: Single callback bringup or teardown selector
48 * @cb_state: The state for a single callback (install/uninstall)
49 * @result: Result of the operation
50 * @done_up: Signal completion to the issuer of the task for cpu-up
51 * @done_down: Signal completion to the issuer of the task for cpu-down
53 struct cpuhp_cpu_state
{
54 enum cpuhp_state state
;
55 enum cpuhp_state target
;
56 enum cpuhp_state fail
;
58 struct task_struct
*thread
;
63 struct hlist_node
*node
;
64 struct hlist_node
*last
;
65 enum cpuhp_state cb_state
;
67 struct completion done_up
;
68 struct completion done_down
;
72 static DEFINE_PER_CPU(struct cpuhp_cpu_state
, cpuhp_state
) = {
73 .fail
= CPUHP_INVALID
,
76 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
77 static struct lockdep_map cpuhp_state_up_map
=
78 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map
);
79 static struct lockdep_map cpuhp_state_down_map
=
80 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map
);
83 static void inline cpuhp_lock_acquire(bool bringup
)
85 lock_map_acquire(bringup
? &cpuhp_state_up_map
: &cpuhp_state_down_map
);
88 static void inline cpuhp_lock_release(bool bringup
)
90 lock_map_release(bringup
? &cpuhp_state_up_map
: &cpuhp_state_down_map
);
94 static void inline cpuhp_lock_acquire(bool bringup
) { }
95 static void inline cpuhp_lock_release(bool bringup
) { }
100 * cpuhp_step - Hotplug state machine step
101 * @name: Name of the step
102 * @startup: Startup function of the step
103 * @teardown: Teardown function of the step
104 * @skip_onerr: Do not invoke the functions on error rollback
105 * Will go away once the notifiers are gone
106 * @cant_stop: Bringup/teardown can't be stopped at this step
111 int (*single
)(unsigned int cpu
);
112 int (*multi
)(unsigned int cpu
,
113 struct hlist_node
*node
);
116 int (*single
)(unsigned int cpu
);
117 int (*multi
)(unsigned int cpu
,
118 struct hlist_node
*node
);
120 struct hlist_head list
;
126 static DEFINE_MUTEX(cpuhp_state_mutex
);
127 static struct cpuhp_step cpuhp_bp_states
[];
128 static struct cpuhp_step cpuhp_ap_states
[];
130 static bool cpuhp_is_ap_state(enum cpuhp_state state
)
133 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
134 * purposes as that state is handled explicitly in cpu_down.
136 return state
> CPUHP_BRINGUP_CPU
&& state
!= CPUHP_TEARDOWN_CPU
;
139 static struct cpuhp_step
*cpuhp_get_step(enum cpuhp_state state
)
141 struct cpuhp_step
*sp
;
143 sp
= cpuhp_is_ap_state(state
) ? cpuhp_ap_states
: cpuhp_bp_states
;
148 * cpuhp_invoke_callback _ Invoke the callbacks for a given state
149 * @cpu: The cpu for which the callback should be invoked
150 * @state: The state to do callbacks for
151 * @bringup: True if the bringup callback should be invoked
152 * @node: For multi-instance, do a single entry callback for install/remove
153 * @lastp: For multi-instance rollback, remember how far we got
155 * Called from cpu hotplug and from the state register machinery.
157 static int cpuhp_invoke_callback(unsigned int cpu
, enum cpuhp_state state
,
158 bool bringup
, struct hlist_node
*node
,
159 struct hlist_node
**lastp
)
161 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
162 struct cpuhp_step
*step
= cpuhp_get_step(state
);
163 int (*cbm
)(unsigned int cpu
, struct hlist_node
*node
);
164 int (*cb
)(unsigned int cpu
);
167 if (st
->fail
== state
) {
168 st
->fail
= CPUHP_INVALID
;
170 if (!(bringup
? step
->startup
.single
: step
->teardown
.single
))
176 if (!step
->multi_instance
) {
177 WARN_ON_ONCE(lastp
&& *lastp
);
178 cb
= bringup
? step
->startup
.single
: step
->teardown
.single
;
181 trace_cpuhp_enter(cpu
, st
->target
, state
, cb
);
183 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
186 cbm
= bringup
? step
->startup
.multi
: step
->teardown
.multi
;
190 /* Single invocation for instance add/remove */
192 WARN_ON_ONCE(lastp
&& *lastp
);
193 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
194 ret
= cbm(cpu
, node
);
195 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
199 /* State transition. Invoke on all instances */
201 hlist_for_each(node
, &step
->list
) {
202 if (lastp
&& node
== *lastp
)
205 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
206 ret
= cbm(cpu
, node
);
207 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
221 /* Rollback the instances if one failed */
222 cbm
= !bringup
? step
->startup
.multi
: step
->teardown
.multi
;
226 hlist_for_each(node
, &step
->list
) {
230 trace_cpuhp_multi_enter(cpu
, st
->target
, state
, cbm
, node
);
231 ret
= cbm(cpu
, node
);
232 trace_cpuhp_exit(cpu
, st
->state
, state
, ret
);
234 * Rollback must not fail,
242 static inline void wait_for_ap_thread(struct cpuhp_cpu_state
*st
, bool bringup
)
244 struct completion
*done
= bringup
? &st
->done_up
: &st
->done_down
;
245 wait_for_completion(done
);
248 static inline void complete_ap_thread(struct cpuhp_cpu_state
*st
, bool bringup
)
250 struct completion
*done
= bringup
? &st
->done_up
: &st
->done_down
;
255 * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
257 static bool cpuhp_is_atomic_state(enum cpuhp_state state
)
259 return CPUHP_AP_IDLE_DEAD
<= state
&& state
< CPUHP_AP_ONLINE
;
262 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
263 static DEFINE_MUTEX(cpu_add_remove_lock
);
264 bool cpuhp_tasks_frozen
;
265 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen
);
268 * The following two APIs (cpu_maps_update_begin/done) must be used when
269 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
271 void cpu_maps_update_begin(void)
273 mutex_lock(&cpu_add_remove_lock
);
276 void cpu_maps_update_done(void)
278 mutex_unlock(&cpu_add_remove_lock
);
282 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
283 * Should always be manipulated under cpu_add_remove_lock
285 static int cpu_hotplug_disabled
;
287 #ifdef CONFIG_HOTPLUG_CPU
289 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock
);
291 void cpus_read_lock(void)
293 percpu_down_read(&cpu_hotplug_lock
);
295 EXPORT_SYMBOL_GPL(cpus_read_lock
);
297 void cpus_read_unlock(void)
299 percpu_up_read(&cpu_hotplug_lock
);
301 EXPORT_SYMBOL_GPL(cpus_read_unlock
);
303 void cpus_write_lock(void)
305 percpu_down_write(&cpu_hotplug_lock
);
308 void cpus_write_unlock(void)
310 percpu_up_write(&cpu_hotplug_lock
);
313 void lockdep_assert_cpus_held(void)
315 percpu_rwsem_assert_held(&cpu_hotplug_lock
);
319 * Wait for currently running CPU hotplug operations to complete (if any) and
320 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
321 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
322 * hotplug path before performing hotplug operations. So acquiring that lock
323 * guarantees mutual exclusion from any currently running hotplug operations.
325 void cpu_hotplug_disable(void)
327 cpu_maps_update_begin();
328 cpu_hotplug_disabled
++;
329 cpu_maps_update_done();
331 EXPORT_SYMBOL_GPL(cpu_hotplug_disable
);
333 static void __cpu_hotplug_enable(void)
335 if (WARN_ONCE(!cpu_hotplug_disabled
, "Unbalanced cpu hotplug enable\n"))
337 cpu_hotplug_disabled
--;
340 void cpu_hotplug_enable(void)
342 cpu_maps_update_begin();
343 __cpu_hotplug_enable();
344 cpu_maps_update_done();
346 EXPORT_SYMBOL_GPL(cpu_hotplug_enable
);
347 #endif /* CONFIG_HOTPLUG_CPU */
349 static inline enum cpuhp_state
350 cpuhp_set_state(struct cpuhp_cpu_state
*st
, enum cpuhp_state target
)
352 enum cpuhp_state prev_state
= st
->state
;
354 st
->rollback
= false;
359 st
->bringup
= st
->state
< target
;
365 cpuhp_reset_state(struct cpuhp_cpu_state
*st
, enum cpuhp_state prev_state
)
370 * If we have st->last we need to undo partial multi_instance of this
371 * state first. Otherwise start undo at the previous state.
380 st
->target
= prev_state
;
381 st
->bringup
= !st
->bringup
;
384 /* Regular hotplug invocation of the AP hotplug thread */
385 static void __cpuhp_kick_ap(struct cpuhp_cpu_state
*st
)
387 if (!st
->single
&& st
->state
== st
->target
)
392 * Make sure the above stores are visible before should_run becomes
393 * true. Paired with the mb() above in cpuhp_thread_fun()
396 st
->should_run
= true;
397 wake_up_process(st
->thread
);
398 wait_for_ap_thread(st
, st
->bringup
);
401 static int cpuhp_kick_ap(struct cpuhp_cpu_state
*st
, enum cpuhp_state target
)
403 enum cpuhp_state prev_state
;
406 prev_state
= cpuhp_set_state(st
, target
);
408 if ((ret
= st
->result
)) {
409 cpuhp_reset_state(st
, prev_state
);
416 static int bringup_wait_for_ap(unsigned int cpu
)
418 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
420 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
421 wait_for_ap_thread(st
, true);
422 if (WARN_ON_ONCE((!cpu_online(cpu
))))
425 /* Unpark the stopper thread and the hotplug thread of the target cpu */
426 stop_machine_unpark(cpu
);
427 kthread_unpark(st
->thread
);
429 if (st
->target
<= CPUHP_AP_ONLINE_IDLE
)
432 return cpuhp_kick_ap(st
, st
->target
);
435 static int bringup_cpu(unsigned int cpu
)
437 struct task_struct
*idle
= idle_thread_get(cpu
);
441 * Some architectures have to walk the irq descriptors to
442 * setup the vector space for the cpu which comes online.
443 * Prevent irq alloc/free across the bringup.
447 /* Arch-specific enabling code. */
448 ret
= __cpu_up(cpu
, idle
);
452 return bringup_wait_for_ap(cpu
);
456 * Hotplug state machine related functions
459 static void undo_cpu_up(unsigned int cpu
, struct cpuhp_cpu_state
*st
)
461 for (st
->state
--; st
->state
> st
->target
; st
->state
--) {
462 struct cpuhp_step
*step
= cpuhp_get_step(st
->state
);
464 if (!step
->skip_onerr
)
465 cpuhp_invoke_callback(cpu
, st
->state
, false, NULL
, NULL
);
469 static int cpuhp_up_callbacks(unsigned int cpu
, struct cpuhp_cpu_state
*st
,
470 enum cpuhp_state target
)
472 enum cpuhp_state prev_state
= st
->state
;
475 while (st
->state
< target
) {
477 ret
= cpuhp_invoke_callback(cpu
, st
->state
, true, NULL
, NULL
);
479 st
->target
= prev_state
;
480 undo_cpu_up(cpu
, st
);
488 * The cpu hotplug threads manage the bringup and teardown of the cpus
490 static void cpuhp_create(unsigned int cpu
)
492 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
494 init_completion(&st
->done_up
);
495 init_completion(&st
->done_down
);
498 static int cpuhp_should_run(unsigned int cpu
)
500 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
502 return st
->should_run
;
506 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
507 * callbacks when a state gets [un]installed at runtime.
509 * Each invocation of this function by the smpboot thread does a single AP
512 * It has 3 modes of operation:
513 * - single: runs st->cb_state
514 * - up: runs ++st->state, while st->state < st->target
515 * - down: runs st->state--, while st->state > st->target
517 * When complete or on error, should_run is cleared and the completion is fired.
519 static void cpuhp_thread_fun(unsigned int cpu
)
521 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
522 bool bringup
= st
->bringup
;
523 enum cpuhp_state state
;
526 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
527 * that if we see ->should_run we also see the rest of the state.
531 if (WARN_ON_ONCE(!st
->should_run
))
534 cpuhp_lock_acquire(bringup
);
537 state
= st
->cb_state
;
538 st
->should_run
= false;
543 st
->should_run
= (st
->state
< st
->target
);
544 WARN_ON_ONCE(st
->state
> st
->target
);
548 st
->should_run
= (st
->state
> st
->target
);
549 WARN_ON_ONCE(st
->state
< st
->target
);
553 WARN_ON_ONCE(!cpuhp_is_ap_state(state
));
556 struct cpuhp_step
*step
= cpuhp_get_step(state
);
557 if (step
->skip_onerr
)
561 if (cpuhp_is_atomic_state(state
)) {
563 st
->result
= cpuhp_invoke_callback(cpu
, state
, bringup
, st
->node
, &st
->last
);
567 * STARTING/DYING must not fail!
569 WARN_ON_ONCE(st
->result
);
571 st
->result
= cpuhp_invoke_callback(cpu
, state
, bringup
, st
->node
, &st
->last
);
576 * If we fail on a rollback, we're up a creek without no
577 * paddle, no way forward, no way back. We loose, thanks for
580 WARN_ON_ONCE(st
->rollback
);
581 st
->should_run
= false;
585 cpuhp_lock_release(bringup
);
588 complete_ap_thread(st
, bringup
);
591 /* Invoke a single callback on a remote cpu */
593 cpuhp_invoke_ap_callback(int cpu
, enum cpuhp_state state
, bool bringup
,
594 struct hlist_node
*node
)
596 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
599 if (!cpu_online(cpu
))
602 cpuhp_lock_acquire(false);
603 cpuhp_lock_release(false);
605 cpuhp_lock_acquire(true);
606 cpuhp_lock_release(true);
609 * If we are up and running, use the hotplug thread. For early calls
610 * we invoke the thread function directly.
613 return cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
615 st
->rollback
= false;
619 st
->bringup
= bringup
;
620 st
->cb_state
= state
;
626 * If we failed and did a partial, do a rollback.
628 if ((ret
= st
->result
) && st
->last
) {
630 st
->bringup
= !bringup
;
636 * Clean up the leftovers so the next hotplug operation wont use stale
639 st
->node
= st
->last
= NULL
;
643 static int cpuhp_kick_ap_work(unsigned int cpu
)
645 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
646 enum cpuhp_state prev_state
= st
->state
;
649 cpuhp_lock_acquire(false);
650 cpuhp_lock_release(false);
652 cpuhp_lock_acquire(true);
653 cpuhp_lock_release(true);
655 trace_cpuhp_enter(cpu
, st
->target
, prev_state
, cpuhp_kick_ap_work
);
656 ret
= cpuhp_kick_ap(st
, st
->target
);
657 trace_cpuhp_exit(cpu
, st
->state
, prev_state
, ret
);
662 static struct smp_hotplug_thread cpuhp_threads
= {
663 .store
= &cpuhp_state
.thread
,
664 .create
= &cpuhp_create
,
665 .thread_should_run
= cpuhp_should_run
,
666 .thread_fn
= cpuhp_thread_fun
,
667 .thread_comm
= "cpuhp/%u",
671 void __init
cpuhp_threads_init(void)
673 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads
));
674 kthread_unpark(this_cpu_read(cpuhp_state
.thread
));
677 #ifdef CONFIG_HOTPLUG_CPU
679 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
682 * This function walks all processes, finds a valid mm struct for each one and
683 * then clears a corresponding bit in mm's cpumask. While this all sounds
684 * trivial, there are various non-obvious corner cases, which this function
685 * tries to solve in a safe manner.
687 * Also note that the function uses a somewhat relaxed locking scheme, so it may
688 * be called only for an already offlined CPU.
690 void clear_tasks_mm_cpumask(int cpu
)
692 struct task_struct
*p
;
695 * This function is called after the cpu is taken down and marked
696 * offline, so its not like new tasks will ever get this cpu set in
697 * their mm mask. -- Peter Zijlstra
698 * Thus, we may use rcu_read_lock() here, instead of grabbing
699 * full-fledged tasklist_lock.
701 WARN_ON(cpu_online(cpu
));
703 for_each_process(p
) {
704 struct task_struct
*t
;
707 * Main thread might exit, but other threads may still have
708 * a valid mm. Find one.
710 t
= find_lock_task_mm(p
);
713 cpumask_clear_cpu(cpu
, mm_cpumask(t
->mm
));
719 /* Take this CPU down. */
720 static int take_cpu_down(void *_param
)
722 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
723 enum cpuhp_state target
= max((int)st
->target
, CPUHP_AP_OFFLINE
);
724 int err
, cpu
= smp_processor_id();
727 /* Ensure this CPU doesn't handle any more interrupts. */
728 err
= __cpu_disable();
733 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
734 * do this step again.
736 WARN_ON(st
->state
!= CPUHP_TEARDOWN_CPU
);
738 /* Invoke the former CPU_DYING callbacks */
739 for (; st
->state
> target
; st
->state
--) {
740 ret
= cpuhp_invoke_callback(cpu
, st
->state
, false, NULL
, NULL
);
742 * DYING must not fail!
747 /* Give up timekeeping duties */
748 tick_handover_do_timer();
749 /* Park the stopper thread */
750 stop_machine_park(cpu
);
754 static int takedown_cpu(unsigned int cpu
)
756 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
759 /* Park the smpboot threads */
760 kthread_park(per_cpu_ptr(&cpuhp_state
, cpu
)->thread
);
761 smpboot_park_threads(cpu
);
764 * Prevent irq alloc/free while the dying cpu reorganizes the
765 * interrupt affinities.
770 * So now all preempt/rcu users must observe !cpu_active().
772 err
= stop_machine_cpuslocked(take_cpu_down
, NULL
, cpumask_of(cpu
));
774 /* CPU refused to die */
776 /* Unpark the hotplug thread so we can rollback there */
777 kthread_unpark(per_cpu_ptr(&cpuhp_state
, cpu
)->thread
);
780 BUG_ON(cpu_online(cpu
));
783 * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
784 * runnable tasks from the cpu, there's only the idle task left now
785 * that the migration thread is done doing the stop_machine thing.
787 * Wait for the stop thread to go away.
789 wait_for_ap_thread(st
, false);
790 BUG_ON(st
->state
!= CPUHP_AP_IDLE_DEAD
);
792 /* Interrupts are moved away from the dying cpu, reenable alloc/free */
795 hotplug_cpu__broadcast_tick_pull(cpu
);
796 /* This actually kills the CPU. */
799 tick_cleanup_dead_cpu(cpu
);
800 rcutree_migrate_callbacks(cpu
);
804 static void cpuhp_complete_idle_dead(void *arg
)
806 struct cpuhp_cpu_state
*st
= arg
;
808 complete_ap_thread(st
, false);
811 void cpuhp_report_idle_dead(void)
813 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
815 BUG_ON(st
->state
!= CPUHP_AP_OFFLINE
);
816 rcu_report_dead(smp_processor_id());
817 st
->state
= CPUHP_AP_IDLE_DEAD
;
819 * We cannot call complete after rcu_report_dead() so we delegate it
822 smp_call_function_single(cpumask_first(cpu_online_mask
),
823 cpuhp_complete_idle_dead
, st
, 0);
826 static void undo_cpu_down(unsigned int cpu
, struct cpuhp_cpu_state
*st
)
828 for (st
->state
++; st
->state
< st
->target
; st
->state
++) {
829 struct cpuhp_step
*step
= cpuhp_get_step(st
->state
);
831 if (!step
->skip_onerr
)
832 cpuhp_invoke_callback(cpu
, st
->state
, true, NULL
, NULL
);
836 static int cpuhp_down_callbacks(unsigned int cpu
, struct cpuhp_cpu_state
*st
,
837 enum cpuhp_state target
)
839 enum cpuhp_state prev_state
= st
->state
;
842 for (; st
->state
> target
; st
->state
--) {
843 ret
= cpuhp_invoke_callback(cpu
, st
->state
, false, NULL
, NULL
);
845 st
->target
= prev_state
;
846 undo_cpu_down(cpu
, st
);
853 /* Requires cpu_add_remove_lock to be held */
854 static int __ref
_cpu_down(unsigned int cpu
, int tasks_frozen
,
855 enum cpuhp_state target
)
857 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
858 int prev_state
, ret
= 0;
860 if (num_online_cpus() == 1)
863 if (!cpu_present(cpu
))
868 cpuhp_tasks_frozen
= tasks_frozen
;
870 prev_state
= cpuhp_set_state(st
, target
);
872 * If the current CPU state is in the range of the AP hotplug thread,
873 * then we need to kick the thread.
875 if (st
->state
> CPUHP_TEARDOWN_CPU
) {
876 st
->target
= max((int)target
, CPUHP_TEARDOWN_CPU
);
877 ret
= cpuhp_kick_ap_work(cpu
);
879 * The AP side has done the error rollback already. Just
880 * return the error code..
886 * We might have stopped still in the range of the AP hotplug
887 * thread. Nothing to do anymore.
889 if (st
->state
> CPUHP_TEARDOWN_CPU
)
895 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
896 * to do the further cleanups.
898 ret
= cpuhp_down_callbacks(cpu
, st
, target
);
899 if (ret
&& st
->state
> CPUHP_TEARDOWN_CPU
&& st
->state
< prev_state
) {
900 cpuhp_reset_state(st
, prev_state
);
907 * Do post unplug cleanup. This is still protected against
908 * concurrent CPU hotplug via cpu_add_remove_lock.
910 lockup_detector_cleanup();
914 static int do_cpu_down(unsigned int cpu
, enum cpuhp_state target
)
918 cpu_maps_update_begin();
920 if (cpu_hotplug_disabled
) {
925 err
= _cpu_down(cpu
, 0, target
);
928 cpu_maps_update_done();
932 int cpu_down(unsigned int cpu
)
934 return do_cpu_down(cpu
, CPUHP_OFFLINE
);
936 EXPORT_SYMBOL(cpu_down
);
939 #define takedown_cpu NULL
940 #endif /*CONFIG_HOTPLUG_CPU*/
943 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
944 * @cpu: cpu that just started
946 * It must be called by the arch code on the new cpu, before the new cpu
947 * enables interrupts and before the "boot" cpu returns from __cpu_up().
949 void notify_cpu_starting(unsigned int cpu
)
951 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
952 enum cpuhp_state target
= min((int)st
->target
, CPUHP_AP_ONLINE
);
955 rcu_cpu_starting(cpu
); /* Enables RCU usage on this CPU. */
956 while (st
->state
< target
) {
958 ret
= cpuhp_invoke_callback(cpu
, st
->state
, true, NULL
, NULL
);
960 * STARTING must not fail!
967 * Called from the idle task. Wake up the controlling task which brings the
968 * stopper and the hotplug thread of the upcoming CPU up and then delegates
969 * the rest of the online bringup to the hotplug thread.
971 void cpuhp_online_idle(enum cpuhp_state state
)
973 struct cpuhp_cpu_state
*st
= this_cpu_ptr(&cpuhp_state
);
975 /* Happens for the boot cpu */
976 if (state
!= CPUHP_AP_ONLINE_IDLE
)
979 st
->state
= CPUHP_AP_ONLINE_IDLE
;
980 complete_ap_thread(st
, true);
983 /* Requires cpu_add_remove_lock to be held */
984 static int _cpu_up(unsigned int cpu
, int tasks_frozen
, enum cpuhp_state target
)
986 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
987 struct task_struct
*idle
;
992 if (!cpu_present(cpu
)) {
998 * The caller of do_cpu_up might have raced with another
999 * caller. Ignore it for now.
1001 if (st
->state
>= target
)
1004 if (st
->state
== CPUHP_OFFLINE
) {
1005 /* Let it fail before we try to bring the cpu up */
1006 idle
= idle_thread_get(cpu
);
1008 ret
= PTR_ERR(idle
);
1013 cpuhp_tasks_frozen
= tasks_frozen
;
1015 cpuhp_set_state(st
, target
);
1017 * If the current CPU state is in the range of the AP hotplug thread,
1018 * then we need to kick the thread once more.
1020 if (st
->state
> CPUHP_BRINGUP_CPU
) {
1021 ret
= cpuhp_kick_ap_work(cpu
);
1023 * The AP side has done the error rollback already. Just
1024 * return the error code..
1031 * Try to reach the target state. We max out on the BP at
1032 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1033 * responsible for bringing it up to the target state.
1035 target
= min((int)target
, CPUHP_BRINGUP_CPU
);
1036 ret
= cpuhp_up_callbacks(cpu
, st
, target
);
1038 cpus_write_unlock();
1042 static int do_cpu_up(unsigned int cpu
, enum cpuhp_state target
)
1046 if (!cpu_possible(cpu
)) {
1047 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1049 #if defined(CONFIG_IA64)
1050 pr_err("please check additional_cpus= boot parameter\n");
1055 err
= try_online_node(cpu_to_node(cpu
));
1059 cpu_maps_update_begin();
1061 if (cpu_hotplug_disabled
) {
1066 err
= _cpu_up(cpu
, 0, target
);
1068 cpu_maps_update_done();
1072 int cpu_up(unsigned int cpu
)
1074 return do_cpu_up(cpu
, CPUHP_ONLINE
);
1076 EXPORT_SYMBOL_GPL(cpu_up
);
1078 #ifdef CONFIG_PM_SLEEP_SMP
1079 static cpumask_var_t frozen_cpus
;
1081 int freeze_secondary_cpus(int primary
)
1085 cpu_maps_update_begin();
1086 if (!cpu_online(primary
))
1087 primary
= cpumask_first(cpu_online_mask
);
1089 * We take down all of the non-boot CPUs in one shot to avoid races
1090 * with the userspace trying to use the CPU hotplug at the same time
1092 cpumask_clear(frozen_cpus
);
1094 pr_info("Disabling non-boot CPUs ...\n");
1095 for_each_online_cpu(cpu
) {
1098 trace_suspend_resume(TPS("CPU_OFF"), cpu
, true);
1099 error
= _cpu_down(cpu
, 1, CPUHP_OFFLINE
);
1100 trace_suspend_resume(TPS("CPU_OFF"), cpu
, false);
1102 cpumask_set_cpu(cpu
, frozen_cpus
);
1104 pr_err("Error taking CPU%d down: %d\n", cpu
, error
);
1110 BUG_ON(num_online_cpus() > 1);
1112 pr_err("Non-boot CPUs are not disabled\n");
1115 * Make sure the CPUs won't be enabled by someone else. We need to do
1116 * this even in case of failure as all disable_nonboot_cpus() users are
1117 * supposed to do enable_nonboot_cpus() on the failure path.
1119 cpu_hotplug_disabled
++;
1121 cpu_maps_update_done();
1125 void __weak
arch_enable_nonboot_cpus_begin(void)
1129 void __weak
arch_enable_nonboot_cpus_end(void)
1133 void enable_nonboot_cpus(void)
1136 struct device
*cpu_device
;
1138 /* Allow everyone to use the CPU hotplug again */
1139 cpu_maps_update_begin();
1140 __cpu_hotplug_enable();
1141 if (cpumask_empty(frozen_cpus
))
1144 pr_info("Enabling non-boot CPUs ...\n");
1146 arch_enable_nonboot_cpus_begin();
1148 for_each_cpu(cpu
, frozen_cpus
) {
1149 trace_suspend_resume(TPS("CPU_ON"), cpu
, true);
1150 error
= _cpu_up(cpu
, 1, CPUHP_ONLINE
);
1151 trace_suspend_resume(TPS("CPU_ON"), cpu
, false);
1153 pr_info("CPU%d is up\n", cpu
);
1154 cpu_device
= get_cpu_device(cpu
);
1156 pr_err("%s: failed to get cpu%d device\n",
1159 kobject_uevent(&cpu_device
->kobj
, KOBJ_ONLINE
);
1162 pr_warn("Error taking CPU%d up: %d\n", cpu
, error
);
1165 arch_enable_nonboot_cpus_end();
1167 cpumask_clear(frozen_cpus
);
1169 cpu_maps_update_done();
1172 static int __init
alloc_frozen_cpus(void)
1174 if (!alloc_cpumask_var(&frozen_cpus
, GFP_KERNEL
|__GFP_ZERO
))
1178 core_initcall(alloc_frozen_cpus
);
1181 * When callbacks for CPU hotplug notifications are being executed, we must
1182 * ensure that the state of the system with respect to the tasks being frozen
1183 * or not, as reported by the notification, remains unchanged *throughout the
1184 * duration* of the execution of the callbacks.
1185 * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1187 * This synchronization is implemented by mutually excluding regular CPU
1188 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1189 * Hibernate notifications.
1192 cpu_hotplug_pm_callback(struct notifier_block
*nb
,
1193 unsigned long action
, void *ptr
)
1197 case PM_SUSPEND_PREPARE
:
1198 case PM_HIBERNATION_PREPARE
:
1199 cpu_hotplug_disable();
1202 case PM_POST_SUSPEND
:
1203 case PM_POST_HIBERNATION
:
1204 cpu_hotplug_enable();
1215 static int __init
cpu_hotplug_pm_sync_init(void)
1218 * cpu_hotplug_pm_callback has higher priority than x86
1219 * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1220 * to disable cpu hotplug to avoid cpu hotplug race.
1222 pm_notifier(cpu_hotplug_pm_callback
, 0);
1225 core_initcall(cpu_hotplug_pm_sync_init
);
1227 #endif /* CONFIG_PM_SLEEP_SMP */
1231 #endif /* CONFIG_SMP */
1233 /* Boot processor state steps */
1234 static struct cpuhp_step cpuhp_bp_states
[] = {
1237 .startup
.single
= NULL
,
1238 .teardown
.single
= NULL
,
1241 [CPUHP_CREATE_THREADS
]= {
1242 .name
= "threads:prepare",
1243 .startup
.single
= smpboot_create_threads
,
1244 .teardown
.single
= NULL
,
1247 [CPUHP_PERF_PREPARE
] = {
1248 .name
= "perf:prepare",
1249 .startup
.single
= perf_event_init_cpu
,
1250 .teardown
.single
= perf_event_exit_cpu
,
1252 [CPUHP_WORKQUEUE_PREP
] = {
1253 .name
= "workqueue:prepare",
1254 .startup
.single
= workqueue_prepare_cpu
,
1255 .teardown
.single
= NULL
,
1257 [CPUHP_HRTIMERS_PREPARE
] = {
1258 .name
= "hrtimers:prepare",
1259 .startup
.single
= hrtimers_prepare_cpu
,
1260 .teardown
.single
= hrtimers_dead_cpu
,
1262 [CPUHP_SMPCFD_PREPARE
] = {
1263 .name
= "smpcfd:prepare",
1264 .startup
.single
= smpcfd_prepare_cpu
,
1265 .teardown
.single
= smpcfd_dead_cpu
,
1267 [CPUHP_RELAY_PREPARE
] = {
1268 .name
= "relay:prepare",
1269 .startup
.single
= relay_prepare_cpu
,
1270 .teardown
.single
= NULL
,
1272 [CPUHP_SLAB_PREPARE
] = {
1273 .name
= "slab:prepare",
1274 .startup
.single
= slab_prepare_cpu
,
1275 .teardown
.single
= slab_dead_cpu
,
1277 [CPUHP_RCUTREE_PREP
] = {
1278 .name
= "RCU/tree:prepare",
1279 .startup
.single
= rcutree_prepare_cpu
,
1280 .teardown
.single
= rcutree_dead_cpu
,
1283 * On the tear-down path, timers_dead_cpu() must be invoked
1284 * before blk_mq_queue_reinit_notify() from notify_dead(),
1285 * otherwise a RCU stall occurs.
1287 [CPUHP_TIMERS_PREPARE
] = {
1288 .name
= "timers:dead",
1289 .startup
.single
= timers_prepare_cpu
,
1290 .teardown
.single
= timers_dead_cpu
,
1292 /* Kicks the plugged cpu into life */
1293 [CPUHP_BRINGUP_CPU
] = {
1294 .name
= "cpu:bringup",
1295 .startup
.single
= bringup_cpu
,
1296 .teardown
.single
= NULL
,
1300 * Handled on controll processor until the plugged processor manages
1303 [CPUHP_TEARDOWN_CPU
] = {
1304 .name
= "cpu:teardown",
1305 .startup
.single
= NULL
,
1306 .teardown
.single
= takedown_cpu
,
1310 [CPUHP_BRINGUP_CPU
] = { },
1314 /* Application processor state steps */
1315 static struct cpuhp_step cpuhp_ap_states
[] = {
1317 /* Final state before CPU kills itself */
1318 [CPUHP_AP_IDLE_DEAD
] = {
1319 .name
= "idle:dead",
1322 * Last state before CPU enters the idle loop to die. Transient state
1323 * for synchronization.
1325 [CPUHP_AP_OFFLINE
] = {
1326 .name
= "ap:offline",
1329 /* First state is scheduler control. Interrupts are disabled */
1330 [CPUHP_AP_SCHED_STARTING
] = {
1331 .name
= "sched:starting",
1332 .startup
.single
= sched_cpu_starting
,
1333 .teardown
.single
= sched_cpu_dying
,
1335 [CPUHP_AP_RCUTREE_DYING
] = {
1336 .name
= "RCU/tree:dying",
1337 .startup
.single
= NULL
,
1338 .teardown
.single
= rcutree_dying_cpu
,
1340 [CPUHP_AP_SMPCFD_DYING
] = {
1341 .name
= "smpcfd:dying",
1342 .startup
.single
= NULL
,
1343 .teardown
.single
= smpcfd_dying_cpu
,
1345 /* Entry state on starting. Interrupts enabled from here on. Transient
1346 * state for synchronsization */
1347 [CPUHP_AP_ONLINE
] = {
1348 .name
= "ap:online",
1350 /* Handle smpboot threads park/unpark */
1351 [CPUHP_AP_SMPBOOT_THREADS
] = {
1352 .name
= "smpboot/threads:online",
1353 .startup
.single
= smpboot_unpark_threads
,
1354 .teardown
.single
= NULL
,
1356 [CPUHP_AP_IRQ_AFFINITY_ONLINE
] = {
1357 .name
= "irq/affinity:online",
1358 .startup
.single
= irq_affinity_online_cpu
,
1359 .teardown
.single
= NULL
,
1361 [CPUHP_AP_PERF_ONLINE
] = {
1362 .name
= "perf:online",
1363 .startup
.single
= perf_event_init_cpu
,
1364 .teardown
.single
= perf_event_exit_cpu
,
1366 [CPUHP_AP_WORKQUEUE_ONLINE
] = {
1367 .name
= "workqueue:online",
1368 .startup
.single
= workqueue_online_cpu
,
1369 .teardown
.single
= workqueue_offline_cpu
,
1371 [CPUHP_AP_RCUTREE_ONLINE
] = {
1372 .name
= "RCU/tree:online",
1373 .startup
.single
= rcutree_online_cpu
,
1374 .teardown
.single
= rcutree_offline_cpu
,
1378 * The dynamically registered state space is here
1382 /* Last state is scheduler control setting the cpu active */
1383 [CPUHP_AP_ACTIVE
] = {
1384 .name
= "sched:active",
1385 .startup
.single
= sched_cpu_activate
,
1386 .teardown
.single
= sched_cpu_deactivate
,
1390 /* CPU is fully up and running. */
1393 .startup
.single
= NULL
,
1394 .teardown
.single
= NULL
,
1398 /* Sanity check for callbacks */
1399 static int cpuhp_cb_check(enum cpuhp_state state
)
1401 if (state
<= CPUHP_OFFLINE
|| state
>= CPUHP_ONLINE
)
1407 * Returns a free for dynamic slot assignment of the Online state. The states
1408 * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1409 * by having no name assigned.
1411 static int cpuhp_reserve_state(enum cpuhp_state state
)
1413 enum cpuhp_state i
, end
;
1414 struct cpuhp_step
*step
;
1417 case CPUHP_AP_ONLINE_DYN
:
1418 step
= cpuhp_ap_states
+ CPUHP_AP_ONLINE_DYN
;
1419 end
= CPUHP_AP_ONLINE_DYN_END
;
1421 case CPUHP_BP_PREPARE_DYN
:
1422 step
= cpuhp_bp_states
+ CPUHP_BP_PREPARE_DYN
;
1423 end
= CPUHP_BP_PREPARE_DYN_END
;
1429 for (i
= state
; i
<= end
; i
++, step
++) {
1433 WARN(1, "No more dynamic states available for CPU hotplug\n");
1437 static int cpuhp_store_callbacks(enum cpuhp_state state
, const char *name
,
1438 int (*startup
)(unsigned int cpu
),
1439 int (*teardown
)(unsigned int cpu
),
1440 bool multi_instance
)
1442 /* (Un)Install the callbacks for further cpu hotplug operations */
1443 struct cpuhp_step
*sp
;
1447 * If name is NULL, then the state gets removed.
1449 * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1450 * the first allocation from these dynamic ranges, so the removal
1451 * would trigger a new allocation and clear the wrong (already
1452 * empty) state, leaving the callbacks of the to be cleared state
1453 * dangling, which causes wreckage on the next hotplug operation.
1455 if (name
&& (state
== CPUHP_AP_ONLINE_DYN
||
1456 state
== CPUHP_BP_PREPARE_DYN
)) {
1457 ret
= cpuhp_reserve_state(state
);
1462 sp
= cpuhp_get_step(state
);
1463 if (name
&& sp
->name
)
1466 sp
->startup
.single
= startup
;
1467 sp
->teardown
.single
= teardown
;
1469 sp
->multi_instance
= multi_instance
;
1470 INIT_HLIST_HEAD(&sp
->list
);
1474 static void *cpuhp_get_teardown_cb(enum cpuhp_state state
)
1476 return cpuhp_get_step(state
)->teardown
.single
;
1480 * Call the startup/teardown function for a step either on the AP or
1481 * on the current CPU.
1483 static int cpuhp_issue_call(int cpu
, enum cpuhp_state state
, bool bringup
,
1484 struct hlist_node
*node
)
1486 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
1490 * If there's nothing to do, we done.
1491 * Relies on the union for multi_instance.
1493 if ((bringup
&& !sp
->startup
.single
) ||
1494 (!bringup
&& !sp
->teardown
.single
))
1497 * The non AP bound callbacks can fail on bringup. On teardown
1498 * e.g. module removal we crash for now.
1501 if (cpuhp_is_ap_state(state
))
1502 ret
= cpuhp_invoke_ap_callback(cpu
, state
, bringup
, node
);
1504 ret
= cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
1506 ret
= cpuhp_invoke_callback(cpu
, state
, bringup
, node
, NULL
);
1508 BUG_ON(ret
&& !bringup
);
1513 * Called from __cpuhp_setup_state on a recoverable failure.
1515 * Note: The teardown callbacks for rollback are not allowed to fail!
1517 static void cpuhp_rollback_install(int failedcpu
, enum cpuhp_state state
,
1518 struct hlist_node
*node
)
1522 /* Roll back the already executed steps on the other cpus */
1523 for_each_present_cpu(cpu
) {
1524 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1525 int cpustate
= st
->state
;
1527 if (cpu
>= failedcpu
)
1530 /* Did we invoke the startup call on that cpu ? */
1531 if (cpustate
>= state
)
1532 cpuhp_issue_call(cpu
, state
, false, node
);
1536 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state
,
1537 struct hlist_node
*node
,
1540 struct cpuhp_step
*sp
;
1544 lockdep_assert_cpus_held();
1546 sp
= cpuhp_get_step(state
);
1547 if (sp
->multi_instance
== false)
1550 mutex_lock(&cpuhp_state_mutex
);
1552 if (!invoke
|| !sp
->startup
.multi
)
1556 * Try to call the startup callback for each present cpu
1557 * depending on the hotplug state of the cpu.
1559 for_each_present_cpu(cpu
) {
1560 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1561 int cpustate
= st
->state
;
1563 if (cpustate
< state
)
1566 ret
= cpuhp_issue_call(cpu
, state
, true, node
);
1568 if (sp
->teardown
.multi
)
1569 cpuhp_rollback_install(cpu
, state
, node
);
1575 hlist_add_head(node
, &sp
->list
);
1577 mutex_unlock(&cpuhp_state_mutex
);
1581 int __cpuhp_state_add_instance(enum cpuhp_state state
, struct hlist_node
*node
,
1587 ret
= __cpuhp_state_add_instance_cpuslocked(state
, node
, invoke
);
1591 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance
);
1594 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1595 * @state: The state to setup
1596 * @invoke: If true, the startup function is invoked for cpus where
1597 * cpu state >= @state
1598 * @startup: startup callback function
1599 * @teardown: teardown callback function
1600 * @multi_instance: State is set up for multiple instances which get
1603 * The caller needs to hold cpus read locked while calling this function.
1606 * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1607 * 0 for all other states
1608 * On failure: proper (negative) error code
1610 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state
,
1611 const char *name
, bool invoke
,
1612 int (*startup
)(unsigned int cpu
),
1613 int (*teardown
)(unsigned int cpu
),
1614 bool multi_instance
)
1619 lockdep_assert_cpus_held();
1621 if (cpuhp_cb_check(state
) || !name
)
1624 mutex_lock(&cpuhp_state_mutex
);
1626 ret
= cpuhp_store_callbacks(state
, name
, startup
, teardown
,
1629 dynstate
= state
== CPUHP_AP_ONLINE_DYN
;
1630 if (ret
> 0 && dynstate
) {
1635 if (ret
|| !invoke
|| !startup
)
1639 * Try to call the startup callback for each present cpu
1640 * depending on the hotplug state of the cpu.
1642 for_each_present_cpu(cpu
) {
1643 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1644 int cpustate
= st
->state
;
1646 if (cpustate
< state
)
1649 ret
= cpuhp_issue_call(cpu
, state
, true, NULL
);
1652 cpuhp_rollback_install(cpu
, state
, NULL
);
1653 cpuhp_store_callbacks(state
, NULL
, NULL
, NULL
, false);
1658 mutex_unlock(&cpuhp_state_mutex
);
1660 * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1661 * dynamically allocated state in case of success.
1663 if (!ret
&& dynstate
)
1667 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked
);
1669 int __cpuhp_setup_state(enum cpuhp_state state
,
1670 const char *name
, bool invoke
,
1671 int (*startup
)(unsigned int cpu
),
1672 int (*teardown
)(unsigned int cpu
),
1673 bool multi_instance
)
1678 ret
= __cpuhp_setup_state_cpuslocked(state
, name
, invoke
, startup
,
1679 teardown
, multi_instance
);
1683 EXPORT_SYMBOL(__cpuhp_setup_state
);
1685 int __cpuhp_state_remove_instance(enum cpuhp_state state
,
1686 struct hlist_node
*node
, bool invoke
)
1688 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
1691 BUG_ON(cpuhp_cb_check(state
));
1693 if (!sp
->multi_instance
)
1697 mutex_lock(&cpuhp_state_mutex
);
1699 if (!invoke
|| !cpuhp_get_teardown_cb(state
))
1702 * Call the teardown callback for each present cpu depending
1703 * on the hotplug state of the cpu. This function is not
1704 * allowed to fail currently!
1706 for_each_present_cpu(cpu
) {
1707 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1708 int cpustate
= st
->state
;
1710 if (cpustate
>= state
)
1711 cpuhp_issue_call(cpu
, state
, false, node
);
1716 mutex_unlock(&cpuhp_state_mutex
);
1721 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance
);
1724 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1725 * @state: The state to remove
1726 * @invoke: If true, the teardown function is invoked for cpus where
1727 * cpu state >= @state
1729 * The caller needs to hold cpus read locked while calling this function.
1730 * The teardown callback is currently not allowed to fail. Think
1731 * about module removal!
1733 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state
, bool invoke
)
1735 struct cpuhp_step
*sp
= cpuhp_get_step(state
);
1738 BUG_ON(cpuhp_cb_check(state
));
1740 lockdep_assert_cpus_held();
1742 mutex_lock(&cpuhp_state_mutex
);
1743 if (sp
->multi_instance
) {
1744 WARN(!hlist_empty(&sp
->list
),
1745 "Error: Removing state %d which has instances left.\n",
1750 if (!invoke
|| !cpuhp_get_teardown_cb(state
))
1754 * Call the teardown callback for each present cpu depending
1755 * on the hotplug state of the cpu. This function is not
1756 * allowed to fail currently!
1758 for_each_present_cpu(cpu
) {
1759 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, cpu
);
1760 int cpustate
= st
->state
;
1762 if (cpustate
>= state
)
1763 cpuhp_issue_call(cpu
, state
, false, NULL
);
1766 cpuhp_store_callbacks(state
, NULL
, NULL
, NULL
, false);
1767 mutex_unlock(&cpuhp_state_mutex
);
1769 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked
);
1771 void __cpuhp_remove_state(enum cpuhp_state state
, bool invoke
)
1774 __cpuhp_remove_state_cpuslocked(state
, invoke
);
1777 EXPORT_SYMBOL(__cpuhp_remove_state
);
1779 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1780 static ssize_t
show_cpuhp_state(struct device
*dev
,
1781 struct device_attribute
*attr
, char *buf
)
1783 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1785 return sprintf(buf
, "%d\n", st
->state
);
1787 static DEVICE_ATTR(state
, 0444, show_cpuhp_state
, NULL
);
1789 static ssize_t
write_cpuhp_target(struct device
*dev
,
1790 struct device_attribute
*attr
,
1791 const char *buf
, size_t count
)
1793 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1794 struct cpuhp_step
*sp
;
1797 ret
= kstrtoint(buf
, 10, &target
);
1801 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1802 if (target
< CPUHP_OFFLINE
|| target
> CPUHP_ONLINE
)
1805 if (target
!= CPUHP_OFFLINE
&& target
!= CPUHP_ONLINE
)
1809 ret
= lock_device_hotplug_sysfs();
1813 mutex_lock(&cpuhp_state_mutex
);
1814 sp
= cpuhp_get_step(target
);
1815 ret
= !sp
->name
|| sp
->cant_stop
? -EINVAL
: 0;
1816 mutex_unlock(&cpuhp_state_mutex
);
1820 if (st
->state
< target
)
1821 ret
= do_cpu_up(dev
->id
, target
);
1823 ret
= do_cpu_down(dev
->id
, target
);
1825 unlock_device_hotplug();
1826 return ret
? ret
: count
;
1829 static ssize_t
show_cpuhp_target(struct device
*dev
,
1830 struct device_attribute
*attr
, char *buf
)
1832 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1834 return sprintf(buf
, "%d\n", st
->target
);
1836 static DEVICE_ATTR(target
, 0644, show_cpuhp_target
, write_cpuhp_target
);
1839 static ssize_t
write_cpuhp_fail(struct device
*dev
,
1840 struct device_attribute
*attr
,
1841 const char *buf
, size_t count
)
1843 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1844 struct cpuhp_step
*sp
;
1847 ret
= kstrtoint(buf
, 10, &fail
);
1852 * Cannot fail STARTING/DYING callbacks.
1854 if (cpuhp_is_atomic_state(fail
))
1858 * Cannot fail anything that doesn't have callbacks.
1860 mutex_lock(&cpuhp_state_mutex
);
1861 sp
= cpuhp_get_step(fail
);
1862 if (!sp
->startup
.single
&& !sp
->teardown
.single
)
1864 mutex_unlock(&cpuhp_state_mutex
);
1873 static ssize_t
show_cpuhp_fail(struct device
*dev
,
1874 struct device_attribute
*attr
, char *buf
)
1876 struct cpuhp_cpu_state
*st
= per_cpu_ptr(&cpuhp_state
, dev
->id
);
1878 return sprintf(buf
, "%d\n", st
->fail
);
1881 static DEVICE_ATTR(fail
, 0644, show_cpuhp_fail
, write_cpuhp_fail
);
1883 static struct attribute
*cpuhp_cpu_attrs
[] = {
1884 &dev_attr_state
.attr
,
1885 &dev_attr_target
.attr
,
1886 &dev_attr_fail
.attr
,
1890 static const struct attribute_group cpuhp_cpu_attr_group
= {
1891 .attrs
= cpuhp_cpu_attrs
,
1896 static ssize_t
show_cpuhp_states(struct device
*dev
,
1897 struct device_attribute
*attr
, char *buf
)
1899 ssize_t cur
, res
= 0;
1902 mutex_lock(&cpuhp_state_mutex
);
1903 for (i
= CPUHP_OFFLINE
; i
<= CPUHP_ONLINE
; i
++) {
1904 struct cpuhp_step
*sp
= cpuhp_get_step(i
);
1907 cur
= sprintf(buf
, "%3d: %s\n", i
, sp
->name
);
1912 mutex_unlock(&cpuhp_state_mutex
);
1915 static DEVICE_ATTR(states
, 0444, show_cpuhp_states
, NULL
);
1917 static struct attribute
*cpuhp_cpu_root_attrs
[] = {
1918 &dev_attr_states
.attr
,
1922 static const struct attribute_group cpuhp_cpu_root_attr_group
= {
1923 .attrs
= cpuhp_cpu_root_attrs
,
1928 static int __init
cpuhp_sysfs_init(void)
1932 ret
= sysfs_create_group(&cpu_subsys
.dev_root
->kobj
,
1933 &cpuhp_cpu_root_attr_group
);
1937 for_each_possible_cpu(cpu
) {
1938 struct device
*dev
= get_cpu_device(cpu
);
1942 ret
= sysfs_create_group(&dev
->kobj
, &cpuhp_cpu_attr_group
);
1948 device_initcall(cpuhp_sysfs_init
);
1952 * cpu_bit_bitmap[] is a special, "compressed" data structure that
1953 * represents all NR_CPUS bits binary values of 1<<nr.
1955 * It is used by cpumask_of() to get a constant address to a CPU
1956 * mask value that has a single bit set only.
1959 /* cpu_bit_bitmap[0] is empty - so we can back into it */
1960 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
1961 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1962 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1963 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
1965 const unsigned long cpu_bit_bitmap
[BITS_PER_LONG
+1][BITS_TO_LONGS(NR_CPUS
)] = {
1967 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
1968 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
1969 #if BITS_PER_LONG > 32
1970 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
1971 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
1974 EXPORT_SYMBOL_GPL(cpu_bit_bitmap
);
1976 const DECLARE_BITMAP(cpu_all_bits
, NR_CPUS
) = CPU_BITS_ALL
;
1977 EXPORT_SYMBOL(cpu_all_bits
);
1979 #ifdef CONFIG_INIT_ALL_POSSIBLE
1980 struct cpumask __cpu_possible_mask __read_mostly
1983 struct cpumask __cpu_possible_mask __read_mostly
;
1985 EXPORT_SYMBOL(__cpu_possible_mask
);
1987 struct cpumask __cpu_online_mask __read_mostly
;
1988 EXPORT_SYMBOL(__cpu_online_mask
);
1990 struct cpumask __cpu_present_mask __read_mostly
;
1991 EXPORT_SYMBOL(__cpu_present_mask
);
1993 struct cpumask __cpu_active_mask __read_mostly
;
1994 EXPORT_SYMBOL(__cpu_active_mask
);
1996 void init_cpu_present(const struct cpumask
*src
)
1998 cpumask_copy(&__cpu_present_mask
, src
);
2001 void init_cpu_possible(const struct cpumask
*src
)
2003 cpumask_copy(&__cpu_possible_mask
, src
);
2006 void init_cpu_online(const struct cpumask
*src
)
2008 cpumask_copy(&__cpu_online_mask
, src
);
2012 * Activate the first processor.
2014 void __init
boot_cpu_init(void)
2016 int cpu
= smp_processor_id();
2018 /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2019 set_cpu_online(cpu
, true);
2020 set_cpu_active(cpu
, true);
2021 set_cpu_present(cpu
, true);
2022 set_cpu_possible(cpu
, true);
2025 __boot_cpu_id
= cpu
;
2030 * Must be called _AFTER_ setting up the per_cpu areas
2032 void __init
boot_cpu_state_init(void)
2034 per_cpu_ptr(&cpuhp_state
, smp_processor_id())->state
= CPUHP_ONLINE
;