Commit | Line | Data |
---|---|---|
ffdb5976 | 1 | /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
e5582ca2 RR |
2 | * GPL v2 and any later version. |
3 | */ | |
1da177e4 LT |
4 | #include <linux/cpu.h> |
5 | #include <linux/err.h> | |
ee527cd3 PB |
6 | #include <linux/kthread.h> |
7 | #include <linux/module.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/stop_machine.h> | |
1da177e4 | 10 | #include <linux/syscalls.h> |
a12bb444 BH |
11 | #include <linux/interrupt.h> |
12 | ||
1da177e4 | 13 | #include <asm/atomic.h> |
1da177e4 LT |
14 | #include <asm/uaccess.h> |
15 | ||
ffdb5976 | 16 | /* This controls the threads on each CPU. */ |
1da177e4 | 17 | enum stopmachine_state { |
ffdb5976 RR |
18 | /* Dummy starting state for thread. */ |
19 | STOPMACHINE_NONE, | |
20 | /* Awaiting everyone to be scheduled. */ | |
1da177e4 | 21 | STOPMACHINE_PREPARE, |
ffdb5976 | 22 | /* Disable interrupts. */ |
1da177e4 | 23 | STOPMACHINE_DISABLE_IRQ, |
ffdb5976 | 24 | /* Run the function */ |
5c2aed62 | 25 | STOPMACHINE_RUN, |
ffdb5976 | 26 | /* Exit */ |
1da177e4 LT |
27 | STOPMACHINE_EXIT, |
28 | }; | |
ffdb5976 | 29 | static enum stopmachine_state state; |
1da177e4 | 30 | |
5c2aed62 JB |
31 | struct stop_machine_data { |
32 | int (*fn)(void *); | |
33 | void *data; | |
ffdb5976 RR |
34 | int fnret; |
35 | }; | |
5c2aed62 | 36 | |
ffdb5976 RR |
37 | /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ |
38 | static unsigned int num_threads; | |
39 | static atomic_t thread_ack; | |
ffdb5976 | 40 | static DEFINE_MUTEX(lock); |
1da177e4 | 41 | |
c9583e55 HC |
42 | static struct workqueue_struct *stop_machine_wq; |
43 | static struct stop_machine_data active, idle; | |
44 | static const cpumask_t *active_cpus; | |
45 | static void *stop_machine_work; | |
46 | ||
ffdb5976 | 47 | static void set_state(enum stopmachine_state newstate) |
1da177e4 | 48 | { |
ffdb5976 RR |
49 | /* Reset ack counter. */ |
50 | atomic_set(&thread_ack, num_threads); | |
51 | smp_wmb(); | |
52 | state = newstate; | |
1da177e4 LT |
53 | } |
54 | ||
ffdb5976 RR |
55 | /* Last one to ack a state moves to the next state. */ |
56 | static void ack_state(void) | |
1da177e4 | 57 | { |
c9583e55 HC |
58 | if (atomic_dec_and_test(&thread_ack)) |
59 | set_state(state + 1); | |
1da177e4 LT |
60 | } |
61 | ||
c9583e55 HC |
62 | /* This is the actual function which stops the CPU. It runs |
63 | * in the context of a dedicated stopmachine workqueue. */ | |
64 | static void stop_cpu(struct work_struct *unused) | |
1da177e4 | 65 | { |
ffdb5976 | 66 | enum stopmachine_state curstate = STOPMACHINE_NONE; |
c9583e55 HC |
67 | struct stop_machine_data *smdata = &idle; |
68 | int cpu = smp_processor_id(); | |
69 | ||
70 | if (!active_cpus) { | |
71 | if (cpu == first_cpu(cpu_online_map)) | |
72 | smdata = &active; | |
73 | } else { | |
74 | if (cpu_isset(cpu, *active_cpus)) | |
75 | smdata = &active; | |
76 | } | |
ffdb5976 RR |
77 | /* Simple state machine */ |
78 | do { | |
79 | /* Chill out and ensure we re-read stopmachine_state. */ | |
3401a61e | 80 | cpu_relax(); |
ffdb5976 RR |
81 | if (state != curstate) { |
82 | curstate = state; | |
83 | switch (curstate) { | |
84 | case STOPMACHINE_DISABLE_IRQ: | |
85 | local_irq_disable(); | |
86 | hard_irq_disable(); | |
87 | break; | |
88 | case STOPMACHINE_RUN: | |
89 | /* |= allows error detection if functions on | |
90 | * multiple CPUs. */ | |
91 | smdata->fnret |= smdata->fn(smdata->data); | |
92 | break; | |
93 | default: | |
94 | break; | |
95 | } | |
96 | ack_state(); | |
97 | } | |
98 | } while (curstate != STOPMACHINE_EXIT); | |
1da177e4 | 99 | |
1da177e4 LT |
100 | local_irq_enable(); |
101 | } | |
102 | ||
ffdb5976 RR |
103 | /* Callback for CPUs which aren't supposed to do anything. */ |
104 | static int chill(void *unused) | |
5c2aed62 | 105 | { |
ffdb5976 | 106 | return 0; |
5c2aed62 | 107 | } |
1da177e4 | 108 | |
eeec4fad | 109 | int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
1da177e4 | 110 | { |
c9583e55 HC |
111 | struct work_struct *sm_work; |
112 | int i; | |
ffdb5976 | 113 | |
c9583e55 HC |
114 | /* Set up initial state. */ |
115 | mutex_lock(&lock); | |
116 | num_threads = num_online_cpus(); | |
117 | active_cpus = cpus; | |
ffdb5976 RR |
118 | active.fn = fn; |
119 | active.data = data; | |
120 | active.fnret = 0; | |
121 | idle.fn = chill; | |
122 | idle.data = NULL; | |
123 | ||
ffdb5976 | 124 | set_state(STOPMACHINE_PREPARE); |
1da177e4 | 125 | |
c9583e55 | 126 | /* Schedule the stop_cpu work on all cpus: hold this CPU so one |
ffdb5976 | 127 | * doesn't hit this CPU until we're ready. */ |
eeec4fad | 128 | get_cpu(); |
c9583e55 HC |
129 | for_each_online_cpu(i) { |
130 | sm_work = percpu_ptr(stop_machine_work, i); | |
131 | INIT_WORK(sm_work, stop_cpu); | |
132 | queue_work_on(i, stop_machine_wq, sm_work); | |
133 | } | |
ffdb5976 RR |
134 | /* This will release the thread on our CPU. */ |
135 | put_cpu(); | |
c9583e55 | 136 | flush_workqueue(stop_machine_wq); |
ffdb5976 | 137 | mutex_unlock(&lock); |
ffdb5976 | 138 | return active.fnret; |
1da177e4 LT |
139 | } |
140 | ||
eeec4fad | 141 | int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) |
1da177e4 | 142 | { |
1da177e4 LT |
143 | int ret; |
144 | ||
145 | /* No CPUs can come up or down during this. */ | |
86ef5c9a | 146 | get_online_cpus(); |
eeec4fad | 147 | ret = __stop_machine(fn, data, cpus); |
86ef5c9a | 148 | put_online_cpus(); |
1da177e4 LT |
149 | |
150 | return ret; | |
151 | } | |
eeec4fad | 152 | EXPORT_SYMBOL_GPL(stop_machine); |
c9583e55 HC |
153 | |
154 | static int __init stop_machine_init(void) | |
155 | { | |
156 | stop_machine_wq = create_rt_workqueue("kstop"); | |
157 | stop_machine_work = alloc_percpu(struct work_struct); | |
158 | return 0; | |
159 | } | |
160 | early_initcall(stop_machine_init); |