Commit | Line | Data |
---|---|---|
e5582ca2 RR |
1 | /* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
2 | * GPL v2 and any later version. | |
3 | */ | |
1da177e4 LT |
4 | #include <linux/cpu.h> |
5 | #include <linux/err.h> | |
ee527cd3 PB |
6 | #include <linux/kthread.h> |
7 | #include <linux/module.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/stop_machine.h> | |
1da177e4 | 10 | #include <linux/syscalls.h> |
a12bb444 BH |
11 | #include <linux/interrupt.h> |
12 | ||
1da177e4 | 13 | #include <asm/atomic.h> |
1da177e4 LT |
14 | #include <asm/uaccess.h> |
15 | ||
16 | /* Since we effect priority and affinity (both of which are visible | |
17 | * to, and settable by outside processes) we do indirection via a | |
18 | * kthread. */ | |
19 | ||
20 | /* Thread to stop each CPU in user context. */ | |
21 | enum stopmachine_state { | |
22 | STOPMACHINE_WAIT, | |
23 | STOPMACHINE_PREPARE, | |
24 | STOPMACHINE_DISABLE_IRQ, | |
5c2aed62 | 25 | STOPMACHINE_RUN, |
1da177e4 LT |
26 | STOPMACHINE_EXIT, |
27 | }; | |
28 | ||
5c2aed62 JB |
29 | struct stop_machine_data { |
30 | int (*fn)(void *); | |
31 | void *data; | |
32 | struct completion done; | |
33 | int run_all; | |
34 | } smdata; | |
35 | ||
1da177e4 LT |
36 | static enum stopmachine_state stopmachine_state; |
37 | static unsigned int stopmachine_num_threads; | |
38 | static atomic_t stopmachine_thread_ack; | |
1da177e4 | 39 | |
d8cb7c1d | 40 | static int stopmachine(void *cpu) |
1da177e4 LT |
41 | { |
42 | int irqs_disabled = 0; | |
43 | int prepared = 0; | |
5c2aed62 | 44 | int ran = 0; |
65c01184 | 45 | cpumask_of_cpu_ptr(cpumask, (int)(long)cpu); |
1da177e4 | 46 | |
65c01184 | 47 | set_cpus_allowed_ptr(current, cpumask); |
d8cb7c1d | 48 | |
1da177e4 | 49 | /* Ack: we are alive */ |
d59dd462 | 50 | smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ |
1da177e4 LT |
51 | atomic_inc(&stopmachine_thread_ack); |
52 | ||
53 | /* Simple state machine */ | |
54 | while (stopmachine_state != STOPMACHINE_EXIT) { | |
55 | if (stopmachine_state == STOPMACHINE_DISABLE_IRQ | |
56 | && !irqs_disabled) { | |
57 | local_irq_disable(); | |
a12bb444 | 58 | hard_irq_disable(); |
1da177e4 LT |
59 | irqs_disabled = 1; |
60 | /* Ack: irqs disabled. */ | |
d59dd462 | 61 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
62 | atomic_inc(&stopmachine_thread_ack); |
63 | } else if (stopmachine_state == STOPMACHINE_PREPARE | |
64 | && !prepared) { | |
65 | /* Everyone is in place, hold CPU. */ | |
66 | preempt_disable(); | |
67 | prepared = 1; | |
d59dd462 | 68 | smp_mb(); /* Must read state first. */ |
1da177e4 | 69 | atomic_inc(&stopmachine_thread_ack); |
5c2aed62 JB |
70 | } else if (stopmachine_state == STOPMACHINE_RUN && !ran) { |
71 | smdata.fn(smdata.data); | |
72 | ran = 1; | |
73 | smp_mb(); /* Must read state first. */ | |
74 | atomic_inc(&stopmachine_thread_ack); | |
1da177e4 LT |
75 | } |
76 | /* Yield in first stage: migration threads need to | |
77 | * help our sisters onto their CPUs. */ | |
78 | if (!prepared && !irqs_disabled) | |
79 | yield(); | |
3401a61e | 80 | cpu_relax(); |
1da177e4 LT |
81 | } |
82 | ||
83 | /* Ack: we are exiting. */ | |
d59dd462 | 84 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
85 | atomic_inc(&stopmachine_thread_ack); |
86 | ||
87 | if (irqs_disabled) | |
88 | local_irq_enable(); | |
89 | if (prepared) | |
90 | preempt_enable(); | |
91 | ||
92 | return 0; | |
93 | } | |
94 | ||
95 | /* Change the thread state */ | |
96 | static void stopmachine_set_state(enum stopmachine_state state) | |
97 | { | |
98 | atomic_set(&stopmachine_thread_ack, 0); | |
d59dd462 | 99 | smp_wmb(); |
1da177e4 LT |
100 | stopmachine_state = state; |
101 | while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) | |
102 | cpu_relax(); | |
103 | } | |
104 | ||
105 | static int stop_machine(void) | |
106 | { | |
d8cb7c1d | 107 | int i, ret = 0; |
1da177e4 LT |
108 | |
109 | atomic_set(&stopmachine_thread_ack, 0); | |
110 | stopmachine_num_threads = 0; | |
111 | stopmachine_state = STOPMACHINE_WAIT; | |
112 | ||
113 | for_each_online_cpu(i) { | |
39c715b7 | 114 | if (i == raw_smp_processor_id()) |
1da177e4 | 115 | continue; |
d8cb7c1d AM |
116 | ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL); |
117 | if (ret < 0) | |
1da177e4 LT |
118 | break; |
119 | stopmachine_num_threads++; | |
120 | } | |
121 | ||
122 | /* Wait for them all to come to life. */ | |
3401a61e | 123 | while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) { |
1da177e4 | 124 | yield(); |
3401a61e CB |
125 | cpu_relax(); |
126 | } | |
1da177e4 LT |
127 | |
128 | /* If some failed, kill them all. */ | |
129 | if (ret < 0) { | |
130 | stopmachine_set_state(STOPMACHINE_EXIT); | |
1da177e4 LT |
131 | return ret; |
132 | } | |
133 | ||
1da177e4 | 134 | /* Now they are all started, make them hold the CPUs, ready. */ |
4557398f | 135 | preempt_disable(); |
1da177e4 LT |
136 | stopmachine_set_state(STOPMACHINE_PREPARE); |
137 | ||
138 | /* Make them disable irqs. */ | |
4557398f | 139 | local_irq_disable(); |
a12bb444 | 140 | hard_irq_disable(); |
1da177e4 LT |
141 | stopmachine_set_state(STOPMACHINE_DISABLE_IRQ); |
142 | ||
143 | return 0; | |
144 | } | |
145 | ||
146 | static void restart_machine(void) | |
147 | { | |
148 | stopmachine_set_state(STOPMACHINE_EXIT); | |
149 | local_irq_enable(); | |
4557398f | 150 | preempt_enable_no_resched(); |
1da177e4 LT |
151 | } |
152 | ||
5c2aed62 JB |
153 | static void run_other_cpus(void) |
154 | { | |
155 | stopmachine_set_state(STOPMACHINE_RUN); | |
156 | } | |
1da177e4 LT |
157 | |
158 | static int do_stop(void *_smdata) | |
159 | { | |
160 | struct stop_machine_data *smdata = _smdata; | |
161 | int ret; | |
162 | ||
163 | ret = stop_machine(); | |
164 | if (ret == 0) { | |
165 | ret = smdata->fn(smdata->data); | |
5c2aed62 JB |
166 | if (smdata->run_all) |
167 | run_other_cpus(); | |
1da177e4 LT |
168 | restart_machine(); |
169 | } | |
170 | ||
171 | /* We're done: you can kthread_stop us now */ | |
172 | complete(&smdata->done); | |
173 | ||
174 | /* Wait for kthread_stop */ | |
175 | set_current_state(TASK_INTERRUPTIBLE); | |
176 | while (!kthread_should_stop()) { | |
177 | schedule(); | |
178 | set_current_state(TASK_INTERRUPTIBLE); | |
179 | } | |
180 | __set_current_state(TASK_RUNNING); | |
181 | return ret; | |
182 | } | |
183 | ||
184 | struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, | |
185 | unsigned int cpu) | |
186 | { | |
6c6080f7 | 187 | static DEFINE_MUTEX(stopmachine_mutex); |
1da177e4 LT |
188 | struct stop_machine_data smdata; |
189 | struct task_struct *p; | |
190 | ||
5c2aed62 JB |
191 | mutex_lock(&stopmachine_mutex); |
192 | ||
1da177e4 LT |
193 | smdata.fn = fn; |
194 | smdata.data = data; | |
5c2aed62 | 195 | smdata.run_all = (cpu == ALL_CPUS) ? 1 : 0; |
1da177e4 LT |
196 | init_completion(&smdata.done); |
197 | ||
5c2aed62 | 198 | smp_wmb(); /* make sure other cpus see smdata updates */ |
1da177e4 LT |
199 | |
200 | /* If they don't care which CPU fn runs on, bind to any online one. */ | |
5c2aed62 | 201 | if (cpu == NR_CPUS || cpu == ALL_CPUS) |
39c715b7 | 202 | cpu = raw_smp_processor_id(); |
1da177e4 LT |
203 | |
204 | p = kthread_create(do_stop, &smdata, "kstopmachine"); | |
205 | if (!IS_ERR(p)) { | |
85653af7 ST |
206 | struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; |
207 | ||
208 | /* One high-prio thread per cpu. We'll do this one. */ | |
961ccddd | 209 | sched_setscheduler_nocheck(p, SCHED_FIFO, ¶m); |
1da177e4 LT |
210 | kthread_bind(p, cpu); |
211 | wake_up_process(p); | |
212 | wait_for_completion(&smdata.done); | |
213 | } | |
6c6080f7 | 214 | mutex_unlock(&stopmachine_mutex); |
1da177e4 LT |
215 | return p; |
216 | } | |
217 | ||
218 | int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) | |
219 | { | |
220 | struct task_struct *p; | |
221 | int ret; | |
222 | ||
223 | /* No CPUs can come up or down during this. */ | |
86ef5c9a | 224 | get_online_cpus(); |
1da177e4 LT |
225 | p = __stop_machine_run(fn, data, cpu); |
226 | if (!IS_ERR(p)) | |
227 | ret = kthread_stop(p); | |
228 | else | |
229 | ret = PTR_ERR(p); | |
86ef5c9a | 230 | put_online_cpus(); |
1da177e4 LT |
231 | |
232 | return ret; | |
233 | } | |
ee527cd3 | 234 | EXPORT_SYMBOL_GPL(stop_machine_run); |