Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
cebf589c | 2 | * SMP support for pSeries and BPA machines. |
1da177e4 LT |
3 | * |
4 | * Dave Engebretsen, Peter Bergner, and | |
5 | * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com | |
6 | * | |
7 | * Plus various changes from other IBM teams... | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; either version | |
12 | * 2 of the License, or (at your option) any later version. | |
13 | */ | |
14 | ||
15 | #undef DEBUG | |
16 | ||
17 | #include <linux/config.h> | |
18 | #include <linux/kernel.h> | |
19 | #include <linux/module.h> | |
20 | #include <linux/sched.h> | |
21 | #include <linux/smp.h> | |
22 | #include <linux/interrupt.h> | |
23 | #include <linux/delay.h> | |
24 | #include <linux/init.h> | |
25 | #include <linux/spinlock.h> | |
26 | #include <linux/cache.h> | |
27 | #include <linux/err.h> | |
28 | #include <linux/sysdev.h> | |
29 | #include <linux/cpu.h> | |
30 | ||
31 | #include <asm/ptrace.h> | |
32 | #include <asm/atomic.h> | |
33 | #include <asm/irq.h> | |
34 | #include <asm/page.h> | |
35 | #include <asm/pgtable.h> | |
36 | #include <asm/io.h> | |
37 | #include <asm/prom.h> | |
38 | #include <asm/smp.h> | |
39 | #include <asm/paca.h> | |
40 | #include <asm/time.h> | |
41 | #include <asm/machdep.h> | |
42 | #include <asm/xics.h> | |
43 | #include <asm/cputable.h> | |
1ababe11 | 44 | #include <asm/firmware.h> |
1da177e4 LT |
45 | #include <asm/system.h> |
46 | #include <asm/rtas.h> | |
47 | #include <asm/plpar_wrappers.h> | |
48 | #include <asm/pSeries_reconfig.h> | |
49 | ||
50 | #include "mpic.h" | |
cebf589c | 51 | #include "bpa_iic.h" |
1da177e4 LT |
52 | |
53 | #ifdef DEBUG | |
54 | #define DBG(fmt...) udbg_printf(fmt) | |
55 | #else | |
56 | #define DBG(fmt...) | |
57 | #endif | |
58 | ||
59 | /* | |
60 | * The primary thread of each non-boot processor is recorded here before | |
61 | * smp init. | |
62 | */ | |
63 | static cpumask_t of_spin_map; | |
64 | ||
65 | extern void pSeries_secondary_smp_init(unsigned long); | |
66 | ||
67 | #ifdef CONFIG_HOTPLUG_CPU | |
68 | ||
69 | /* Get state of physical CPU. | |
70 | * Return codes: | |
71 | * 0 - The processor is in the RTAS stopped state | |
72 | * 1 - stop-self is in progress | |
73 | * 2 - The processor is not in the RTAS stopped state | |
74 | * -1 - Hardware Error | |
75 | * -2 - Hardware Busy, Try again later. | |
76 | */ | |
77 | static int query_cpu_stopped(unsigned int pcpu) | |
78 | { | |
79 | int cpu_status; | |
80 | int status, qcss_tok; | |
81 | ||
82 | qcss_tok = rtas_token("query-cpu-stopped-state"); | |
83 | if (qcss_tok == RTAS_UNKNOWN_SERVICE) | |
84 | return -1; | |
85 | status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu); | |
86 | if (status != 0) { | |
87 | printk(KERN_ERR | |
88 | "RTAS query-cpu-stopped-state failed: %i\n", status); | |
89 | return status; | |
90 | } | |
91 | ||
92 | return cpu_status; | |
93 | } | |
94 | ||
95 | int pSeries_cpu_disable(void) | |
96 | { | |
f3705136 ZM |
97 | int cpu = smp_processor_id(); |
98 | ||
99 | cpu_clear(cpu, cpu_online_map); | |
1da177e4 LT |
100 | systemcfg->processorCount--; |
101 | ||
102 | /*fix boot_cpuid here*/ | |
f3705136 | 103 | if (cpu == boot_cpuid) |
1da177e4 LT |
104 | boot_cpuid = any_online_cpu(cpu_online_map); |
105 | ||
106 | /* FIXME: abstract this to not be platform specific later on */ | |
107 | xics_migrate_irqs_away(); | |
108 | return 0; | |
109 | } | |
110 | ||
111 | void pSeries_cpu_die(unsigned int cpu) | |
112 | { | |
113 | int tries; | |
114 | int cpu_status; | |
115 | unsigned int pcpu = get_hard_smp_processor_id(cpu); | |
116 | ||
117 | for (tries = 0; tries < 25; tries++) { | |
118 | cpu_status = query_cpu_stopped(pcpu); | |
119 | if (cpu_status == 0 || cpu_status == -1) | |
120 | break; | |
121 | msleep(200); | |
122 | } | |
123 | if (cpu_status != 0) { | |
124 | printk("Querying DEAD? cpu %i (%i) shows %i\n", | |
125 | cpu, pcpu, cpu_status); | |
126 | } | |
127 | ||
128 | /* Isolation and deallocation are definatly done by | |
129 | * drslot_chrp_cpu. If they were not they would be | |
130 | * done here. Change isolate state to Isolate and | |
131 | * change allocation-state to Unusable. | |
132 | */ | |
133 | paca[cpu].cpu_start = 0; | |
134 | } | |
135 | ||
136 | /* | |
137 | * Update cpu_present_map and paca(s) for a new cpu node. The wrinkle | |
138 | * here is that a cpu device node may represent up to two logical cpus | |
139 | * in the SMT case. We must honor the assumption in other code that | |
140 | * the logical ids for sibling SMT threads x and y are adjacent, such | |
141 | * that x^1 == y and y^1 == x. | |
142 | */ | |
143 | static int pSeries_add_processor(struct device_node *np) | |
144 | { | |
145 | unsigned int cpu; | |
146 | cpumask_t candidate_map, tmp = CPU_MASK_NONE; | |
147 | int err = -ENOSPC, len, nthreads, i; | |
148 | u32 *intserv; | |
149 | ||
150 | intserv = (u32 *)get_property(np, "ibm,ppc-interrupt-server#s", &len); | |
151 | if (!intserv) | |
152 | return 0; | |
153 | ||
154 | nthreads = len / sizeof(u32); | |
155 | for (i = 0; i < nthreads; i++) | |
156 | cpu_set(i, tmp); | |
157 | ||
158 | lock_cpu_hotplug(); | |
159 | ||
160 | BUG_ON(!cpus_subset(cpu_present_map, cpu_possible_map)); | |
161 | ||
162 | /* Get a bitmap of unoccupied slots. */ | |
163 | cpus_xor(candidate_map, cpu_possible_map, cpu_present_map); | |
164 | if (cpus_empty(candidate_map)) { | |
165 | /* If we get here, it most likely means that NR_CPUS is | |
166 | * less than the partition's max processors setting. | |
167 | */ | |
168 | printk(KERN_ERR "Cannot add cpu %s; this system configuration" | |
169 | " supports %d logical cpus.\n", np->full_name, | |
170 | cpus_weight(cpu_possible_map)); | |
171 | goto out_unlock; | |
172 | } | |
173 | ||
174 | while (!cpus_empty(tmp)) | |
175 | if (cpus_subset(tmp, candidate_map)) | |
176 | /* Found a range where we can insert the new cpu(s) */ | |
177 | break; | |
178 | else | |
179 | cpus_shift_left(tmp, tmp, nthreads); | |
180 | ||
181 | if (cpus_empty(tmp)) { | |
182 | printk(KERN_ERR "Unable to find space in cpu_present_map for" | |
183 | " processor %s with %d thread(s)\n", np->name, | |
184 | nthreads); | |
185 | goto out_unlock; | |
186 | } | |
187 | ||
188 | for_each_cpu_mask(cpu, tmp) { | |
189 | BUG_ON(cpu_isset(cpu, cpu_present_map)); | |
190 | cpu_set(cpu, cpu_present_map); | |
191 | set_hard_smp_processor_id(cpu, *intserv++); | |
192 | } | |
193 | err = 0; | |
194 | out_unlock: | |
195 | unlock_cpu_hotplug(); | |
196 | return err; | |
197 | } | |
198 | ||
199 | /* | |
200 | * Update the present map for a cpu node which is going away, and set | |
201 | * the hard id in the paca(s) to -1 to be consistent with boot time | |
202 | * convention for non-present cpus. | |
203 | */ | |
204 | static void pSeries_remove_processor(struct device_node *np) | |
205 | { | |
206 | unsigned int cpu; | |
207 | int len, nthreads, i; | |
208 | u32 *intserv; | |
209 | ||
210 | intserv = (u32 *)get_property(np, "ibm,ppc-interrupt-server#s", &len); | |
211 | if (!intserv) | |
212 | return; | |
213 | ||
214 | nthreads = len / sizeof(u32); | |
215 | ||
216 | lock_cpu_hotplug(); | |
217 | for (i = 0; i < nthreads; i++) { | |
218 | for_each_present_cpu(cpu) { | |
219 | if (get_hard_smp_processor_id(cpu) != intserv[i]) | |
220 | continue; | |
221 | BUG_ON(cpu_online(cpu)); | |
222 | cpu_clear(cpu, cpu_present_map); | |
223 | set_hard_smp_processor_id(cpu, -1); | |
224 | break; | |
225 | } | |
226 | if (cpu == NR_CPUS) | |
227 | printk(KERN_WARNING "Could not find cpu to remove " | |
228 | "with physical id 0x%x\n", intserv[i]); | |
229 | } | |
230 | unlock_cpu_hotplug(); | |
231 | } | |
232 | ||
233 | static int pSeries_smp_notifier(struct notifier_block *nb, unsigned long action, void *node) | |
234 | { | |
235 | int err = NOTIFY_OK; | |
236 | ||
237 | switch (action) { | |
238 | case PSERIES_RECONFIG_ADD: | |
239 | if (pSeries_add_processor(node)) | |
240 | err = NOTIFY_BAD; | |
241 | break; | |
242 | case PSERIES_RECONFIG_REMOVE: | |
243 | pSeries_remove_processor(node); | |
244 | break; | |
245 | default: | |
246 | err = NOTIFY_DONE; | |
247 | break; | |
248 | } | |
249 | return err; | |
250 | } | |
251 | ||
252 | static struct notifier_block pSeries_smp_nb = { | |
253 | .notifier_call = pSeries_smp_notifier, | |
254 | }; | |
255 | ||
256 | #endif /* CONFIG_HOTPLUG_CPU */ | |
257 | ||
258 | /** | |
259 | * smp_startup_cpu() - start the given cpu | |
260 | * | |
261 | * At boot time, there is nothing to do for primary threads which were | |
262 | * started from Open Firmware. For anything else, call RTAS with the | |
263 | * appropriate start location. | |
264 | * | |
265 | * Returns: | |
266 | * 0 - failure | |
267 | * 1 - success | |
268 | */ | |
269 | static inline int __devinit smp_startup_cpu(unsigned int lcpu) | |
270 | { | |
271 | int status; | |
272 | unsigned long start_here = __pa((u32)*((unsigned long *) | |
273 | pSeries_secondary_smp_init)); | |
274 | unsigned int pcpu; | |
275 | ||
276 | if (cpu_isset(lcpu, of_spin_map)) | |
277 | /* Already started by OF and sitting in spin loop */ | |
278 | return 1; | |
279 | ||
280 | pcpu = get_hard_smp_processor_id(lcpu); | |
281 | ||
282 | /* Fixup atomic count: it exited inside IRQ handler. */ | |
283 | paca[lcpu].__current->thread_info->preempt_count = 0; | |
284 | ||
285 | status = rtas_call(rtas_token("start-cpu"), 3, 1, NULL, | |
286 | pcpu, start_here, lcpu); | |
287 | if (status != 0) { | |
288 | printk(KERN_ERR "start-cpu failed: %i\n", status); | |
289 | return 0; | |
290 | } | |
291 | return 1; | |
292 | } | |
293 | ||
cebf589c | 294 | #ifdef CONFIG_XICS |
1da177e4 LT |
295 | static inline void smp_xics_do_message(int cpu, int msg) |
296 | { | |
297 | set_bit(msg, &xics_ipi_message[cpu].value); | |
298 | mb(); | |
299 | xics_cause_IPI(cpu); | |
300 | } | |
301 | ||
302 | static void smp_xics_message_pass(int target, int msg) | |
303 | { | |
304 | unsigned int i; | |
305 | ||
306 | if (target < NR_CPUS) { | |
307 | smp_xics_do_message(target, msg); | |
308 | } else { | |
309 | for_each_online_cpu(i) { | |
310 | if (target == MSG_ALL_BUT_SELF | |
311 | && i == smp_processor_id()) | |
312 | continue; | |
313 | smp_xics_do_message(i, msg); | |
314 | } | |
315 | } | |
316 | } | |
317 | ||
318 | static int __init smp_xics_probe(void) | |
319 | { | |
320 | xics_request_IPIs(); | |
321 | ||
322 | return cpus_weight(cpu_possible_map); | |
323 | } | |
324 | ||
325 | static void __devinit smp_xics_setup_cpu(int cpu) | |
326 | { | |
327 | if (cpu != boot_cpuid) | |
328 | xics_setup_cpu(); | |
329 | ||
1ababe11 | 330 | if (firmware_has_feature(FW_FEATURE_SPLPAR)) |
1da177e4 LT |
331 | vpa_init(cpu); |
332 | ||
333 | cpu_clear(cpu, of_spin_map); | |
334 | ||
1da177e4 | 335 | } |
cebf589c AB |
336 | #endif /* CONFIG_XICS */ |
337 | #ifdef CONFIG_BPA_IIC | |
338 | static void smp_iic_message_pass(int target, int msg) | |
339 | { | |
340 | unsigned int i; | |
341 | ||
342 | if (target < NR_CPUS) { | |
343 | iic_cause_IPI(target, msg); | |
344 | } else { | |
345 | for_each_online_cpu(i) { | |
346 | if (target == MSG_ALL_BUT_SELF | |
347 | && i == smp_processor_id()) | |
348 | continue; | |
349 | iic_cause_IPI(i, msg); | |
350 | } | |
351 | } | |
352 | } | |
353 | ||
354 | static int __init smp_iic_probe(void) | |
355 | { | |
356 | iic_request_IPIs(); | |
357 | ||
358 | return cpus_weight(cpu_possible_map); | |
359 | } | |
360 | ||
361 | static void __devinit smp_iic_setup_cpu(int cpu) | |
362 | { | |
363 | if (cpu != boot_cpuid) | |
364 | iic_setup_cpu(); | |
365 | } | |
366 | #endif /* CONFIG_BPA_IIC */ | |
1da177e4 LT |
367 | |
368 | static DEFINE_SPINLOCK(timebase_lock); | |
369 | static unsigned long timebase = 0; | |
370 | ||
371 | static void __devinit pSeries_give_timebase(void) | |
372 | { | |
373 | spin_lock(&timebase_lock); | |
374 | rtas_call(rtas_token("freeze-time-base"), 0, 1, NULL); | |
375 | timebase = get_tb(); | |
376 | spin_unlock(&timebase_lock); | |
377 | ||
378 | while (timebase) | |
379 | barrier(); | |
380 | rtas_call(rtas_token("thaw-time-base"), 0, 1, NULL); | |
381 | } | |
382 | ||
383 | static void __devinit pSeries_take_timebase(void) | |
384 | { | |
385 | while (!timebase) | |
386 | barrier(); | |
387 | spin_lock(&timebase_lock); | |
388 | set_tb(timebase >> 32, timebase & 0xffffffff); | |
389 | timebase = 0; | |
390 | spin_unlock(&timebase_lock); | |
391 | } | |
392 | ||
393 | static void __devinit smp_pSeries_kick_cpu(int nr) | |
394 | { | |
395 | BUG_ON(nr < 0 || nr >= NR_CPUS); | |
396 | ||
397 | if (!smp_startup_cpu(nr)) | |
398 | return; | |
399 | ||
400 | /* | |
401 | * The processor is currently spinning, waiting for the | |
402 | * cpu_start field to become non-zero After we set cpu_start, | |
403 | * the processor will continue on to secondary_start | |
404 | */ | |
405 | paca[nr].cpu_start = 1; | |
406 | } | |
407 | ||
408 | static int smp_pSeries_cpu_bootable(unsigned int nr) | |
409 | { | |
410 | /* Special case - we inhibit secondary thread startup | |
411 | * during boot if the user requests it. Odd-numbered | |
412 | * cpus are assumed to be secondary threads. | |
413 | */ | |
414 | if (system_state < SYSTEM_RUNNING && | |
0231c290 | 415 | cpu_has_feature(CPU_FTR_SMT) && |
1da177e4 LT |
416 | !smt_enabled_at_boot && nr % 2 != 0) |
417 | return 0; | |
418 | ||
419 | return 1; | |
420 | } | |
cebf589c | 421 | #ifdef CONFIG_MPIC |
1da177e4 LT |
422 | static struct smp_ops_t pSeries_mpic_smp_ops = { |
423 | .message_pass = smp_mpic_message_pass, | |
424 | .probe = smp_mpic_probe, | |
425 | .kick_cpu = smp_pSeries_kick_cpu, | |
426 | .setup_cpu = smp_mpic_setup_cpu, | |
427 | }; | |
cebf589c AB |
428 | #endif |
429 | #ifdef CONFIG_XICS | |
1da177e4 LT |
430 | static struct smp_ops_t pSeries_xics_smp_ops = { |
431 | .message_pass = smp_xics_message_pass, | |
432 | .probe = smp_xics_probe, | |
433 | .kick_cpu = smp_pSeries_kick_cpu, | |
434 | .setup_cpu = smp_xics_setup_cpu, | |
435 | .cpu_bootable = smp_pSeries_cpu_bootable, | |
436 | }; | |
cebf589c AB |
437 | #endif |
438 | #ifdef CONFIG_BPA_IIC | |
439 | static struct smp_ops_t bpa_iic_smp_ops = { | |
440 | .message_pass = smp_iic_message_pass, | |
441 | .probe = smp_iic_probe, | |
442 | .kick_cpu = smp_pSeries_kick_cpu, | |
443 | .setup_cpu = smp_iic_setup_cpu, | |
444 | .cpu_bootable = smp_pSeries_cpu_bootable, | |
445 | }; | |
446 | #endif | |
1da177e4 LT |
447 | |
448 | /* This is called very early */ | |
449 | void __init smp_init_pSeries(void) | |
450 | { | |
451 | int i; | |
452 | ||
453 | DBG(" -> smp_init_pSeries()\n"); | |
454 | ||
cebf589c AB |
455 | switch (ppc64_interrupt_controller) { |
456 | #ifdef CONFIG_MPIC | |
457 | case IC_OPEN_PIC: | |
1da177e4 | 458 | smp_ops = &pSeries_mpic_smp_ops; |
cebf589c AB |
459 | break; |
460 | #endif | |
461 | #ifdef CONFIG_XICS | |
462 | case IC_PPC_XIC: | |
1da177e4 | 463 | smp_ops = &pSeries_xics_smp_ops; |
cebf589c AB |
464 | break; |
465 | #endif | |
466 | #ifdef CONFIG_BPA_IIC | |
467 | case IC_BPA_IIC: | |
468 | smp_ops = &bpa_iic_smp_ops; | |
469 | break; | |
470 | #endif | |
471 | default: | |
472 | panic("Invalid interrupt controller"); | |
473 | } | |
1da177e4 LT |
474 | |
475 | #ifdef CONFIG_HOTPLUG_CPU | |
476 | smp_ops->cpu_disable = pSeries_cpu_disable; | |
477 | smp_ops->cpu_die = pSeries_cpu_die; | |
478 | ||
479 | /* Processors can be added/removed only on LPAR */ | |
480 | if (systemcfg->platform == PLATFORM_PSERIES_LPAR) | |
481 | pSeries_reconfig_notifier_register(&pSeries_smp_nb); | |
482 | #endif | |
483 | ||
484 | /* Mark threads which are still spinning in hold loops. */ | |
0231c290 AB |
485 | if (cpu_has_feature(CPU_FTR_SMT)) { |
486 | for_each_present_cpu(i) { | |
1da177e4 LT |
487 | if (i % 2 == 0) |
488 | /* | |
489 | * Even-numbered logical cpus correspond to | |
490 | * primary threads. | |
491 | */ | |
492 | cpu_set(i, of_spin_map); | |
493 | } | |
0231c290 | 494 | } else { |
1da177e4 | 495 | of_spin_map = cpu_present_map; |
0231c290 | 496 | } |
1da177e4 LT |
497 | |
498 | cpu_clear(boot_cpuid, of_spin_map); | |
499 | ||
500 | /* Non-lpar has additional take/give timebase */ | |
501 | if (rtas_token("freeze-time-base") != RTAS_UNKNOWN_SERVICE) { | |
502 | smp_ops->give_timebase = pSeries_give_timebase; | |
503 | smp_ops->take_timebase = pSeries_take_timebase; | |
504 | } | |
505 | ||
506 | DBG(" <- smp_init_pSeries()\n"); | |
507 | } | |
508 |