Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
69a80d3f | 2 | * SMP support for pSeries machines. |
1da177e4 LT |
3 | * |
4 | * Dave Engebretsen, Peter Bergner, and | |
5 | * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com | |
6 | * | |
7 | * Plus various changes from other IBM teams... | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; either version | |
12 | * 2 of the License, or (at your option) any later version. | |
13 | */ | |
14 | ||
1da177e4 | 15 | |
1da177e4 | 16 | #include <linux/kernel.h> |
1da177e4 LT |
17 | #include <linux/sched.h> |
18 | #include <linux/smp.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <linux/delay.h> | |
21 | #include <linux/init.h> | |
22 | #include <linux/spinlock.h> | |
23 | #include <linux/cache.h> | |
24 | #include <linux/err.h> | |
edbaa603 | 25 | #include <linux/device.h> |
1da177e4 LT |
26 | #include <linux/cpu.h> |
27 | ||
28 | #include <asm/ptrace.h> | |
60063497 | 29 | #include <linux/atomic.h> |
1da177e4 LT |
30 | #include <asm/irq.h> |
31 | #include <asm/page.h> | |
32 | #include <asm/pgtable.h> | |
33 | #include <asm/io.h> | |
34 | #include <asm/prom.h> | |
35 | #include <asm/smp.h> | |
36 | #include <asm/paca.h> | |
1da177e4 | 37 | #include <asm/machdep.h> |
1da177e4 | 38 | #include <asm/cputable.h> |
1ababe11 | 39 | #include <asm/firmware.h> |
1da177e4 | 40 | #include <asm/rtas.h> |
bbeb3f4c | 41 | #include <asm/mpic.h> |
271c3f35 | 42 | #include <asm/vdso_datapage.h> |
8d089085 | 43 | #include <asm/cputhreads.h> |
0b05ac6e | 44 | #include <asm/xics.h> |
e5e84f0a | 45 | #include <asm/dbell.h> |
1da177e4 | 46 | |
a1218720 | 47 | #include "plpar_wrappers.h" |
8feaeca2 | 48 | #include "pseries.h" |
3aa565f5 | 49 | #include "offline_states.h" |
a1218720 | 50 | |
1da177e4 LT |
51 | |
52 | /* | |
6a75a6b8 MM |
53 | * The Primary thread of each non-boot processor was started from the OF client |
54 | * interface by prom_hold_cpus and is spinning on secondary_hold_spinloop. | |
1da177e4 | 55 | */ |
af831e1e | 56 | static cpumask_var_t of_spin_mask; |
1da177e4 | 57 | |
e5e84f0a IM |
58 | /* |
59 | * If we multiplex IPI mechanisms, store the appropriate XICS IPI mechanism here | |
60 | */ | |
61 | static void (*xics_cause_ipi)(int cpu, unsigned long data); | |
62 | ||
f8b67691 MN |
63 | /* Query where a cpu is now. Return codes #defined in plpar_wrappers.h */ |
64 | int smp_query_cpu_stopped(unsigned int pcpu) | |
65 | { | |
66 | int cpu_status, status; | |
67 | int qcss_tok = rtas_token("query-cpu-stopped-state"); | |
68 | ||
69 | if (qcss_tok == RTAS_UNKNOWN_SERVICE) { | |
2d86938a MM |
70 | printk_once(KERN_INFO |
71 | "Firmware doesn't support query-cpu-stopped-state\n"); | |
f8b67691 MN |
72 | return QCSS_HARDWARE_ERROR; |
73 | } | |
74 | ||
75 | status = rtas_call(qcss_tok, 1, 2, &cpu_status, pcpu); | |
76 | if (status != 0) { | |
77 | printk(KERN_ERR | |
78 | "RTAS query-cpu-stopped-state failed: %i\n", status); | |
79 | return status; | |
80 | } | |
81 | ||
82 | return cpu_status; | |
83 | } | |
84 | ||
1da177e4 LT |
85 | /** |
86 | * smp_startup_cpu() - start the given cpu | |
87 | * | |
88 | * At boot time, there is nothing to do for primary threads which were | |
89 | * started from Open Firmware. For anything else, call RTAS with the | |
90 | * appropriate start location. | |
91 | * | |
92 | * Returns: | |
93 | * 0 - failure | |
94 | * 1 - success | |
95 | */ | |
cad5cef6 | 96 | static inline int smp_startup_cpu(unsigned int lcpu) |
1da177e4 LT |
97 | { |
98 | int status; | |
99 | unsigned long start_here = __pa((u32)*((unsigned long *) | |
f39b7a55 | 100 | generic_secondary_smp_init)); |
1da177e4 | 101 | unsigned int pcpu; |
1ed2fd2d | 102 | int start_cpu; |
1da177e4 | 103 | |
af831e1e | 104 | if (cpumask_test_cpu(lcpu, of_spin_mask)) |
1da177e4 LT |
105 | /* Already started by OF and sitting in spin loop */ |
106 | return 1; | |
107 | ||
108 | pcpu = get_hard_smp_processor_id(lcpu); | |
109 | ||
aef40e87 MN |
110 | /* Check to see if the CPU out of FW already for kexec */ |
111 | if (smp_query_cpu_stopped(pcpu) == QCSS_NOT_STOPPED){ | |
af831e1e | 112 | cpumask_set_cpu(lcpu, of_spin_mask); |
aef40e87 MN |
113 | return 1; |
114 | } | |
115 | ||
1da177e4 | 116 | /* Fixup atomic count: it exited inside IRQ handler. */ |
b5e2fc1c | 117 | task_thread_info(paca[lcpu].__current)->preempt_count = 0; |
c60e65d7 | 118 | #ifdef CONFIG_HOTPLUG_CPU |
3aa565f5 GS |
119 | if (get_cpu_current_state(lcpu) == CPU_STATE_INACTIVE) |
120 | goto out; | |
c60e65d7 | 121 | #endif |
1ed2fd2d AB |
122 | /* |
123 | * If the RTAS start-cpu token does not exist then presume the | |
124 | * cpu is already spinning. | |
125 | */ | |
126 | start_cpu = rtas_token("start-cpu"); | |
127 | if (start_cpu == RTAS_UNKNOWN_SERVICE) | |
128 | return 1; | |
129 | ||
496b7a51 | 130 | status = rtas_call(start_cpu, 3, 1, NULL, pcpu, start_here, pcpu); |
1da177e4 LT |
131 | if (status != 0) { |
132 | printk(KERN_ERR "start-cpu failed: %i\n", status); | |
133 | return 0; | |
134 | } | |
1ed2fd2d | 135 | |
c60e65d7 | 136 | #ifdef CONFIG_HOTPLUG_CPU |
3aa565f5 | 137 | out: |
c60e65d7 | 138 | #endif |
1da177e4 LT |
139 | return 1; |
140 | } | |
141 | ||
cad5cef6 | 142 | static void smp_xics_setup_cpu(int cpu) |
1da177e4 LT |
143 | { |
144 | if (cpu != boot_cpuid) | |
145 | xics_setup_cpu(); | |
e5e84f0a IM |
146 | if (cpu_has_feature(CPU_FTR_DBELL)) |
147 | doorbell_setup_this_cpu(); | |
1da177e4 | 148 | |
1ababe11 | 149 | if (firmware_has_feature(FW_FEATURE_SPLPAR)) |
1da177e4 LT |
150 | vpa_init(cpu); |
151 | ||
af831e1e | 152 | cpumask_clear_cpu(cpu, of_spin_mask); |
c60e65d7 | 153 | #ifdef CONFIG_HOTPLUG_CPU |
3aa565f5 GS |
154 | set_cpu_current_state(cpu, CPU_STATE_ONLINE); |
155 | set_default_offline_state(cpu); | |
c60e65d7 | 156 | #endif |
1da177e4 LT |
157 | } |
158 | ||
cad5cef6 | 159 | static int smp_pSeries_kick_cpu(int nr) |
1da177e4 LT |
160 | { |
161 | BUG_ON(nr < 0 || nr >= NR_CPUS); | |
162 | ||
163 | if (!smp_startup_cpu(nr)) | |
de300974 | 164 | return -ENOENT; |
1da177e4 LT |
165 | |
166 | /* | |
167 | * The processor is currently spinning, waiting for the | |
168 | * cpu_start field to become non-zero After we set cpu_start, | |
169 | * the processor will continue on to secondary_start | |
170 | */ | |
171 | paca[nr].cpu_start = 1; | |
c60e65d7 | 172 | #ifdef CONFIG_HOTPLUG_CPU |
3aa565f5 GS |
173 | set_preferred_offline_state(nr, CPU_STATE_ONLINE); |
174 | ||
175 | if (get_cpu_current_state(nr) == CPU_STATE_INACTIVE) { | |
c60e65d7 ME |
176 | long rc; |
177 | unsigned long hcpuid; | |
178 | ||
3aa565f5 GS |
179 | hcpuid = get_hard_smp_processor_id(nr); |
180 | rc = plpar_hcall_norets(H_PROD, hcpuid); | |
181 | if (rc != H_SUCCESS) | |
5a2ad98e JP |
182 | printk(KERN_ERR "Error: Prod to wake up processor %d " |
183 | "Ret= %ld\n", nr, rc); | |
3aa565f5 | 184 | } |
c60e65d7 | 185 | #endif |
de300974 ME |
186 | |
187 | return 0; | |
1da177e4 LT |
188 | } |
189 | ||
190 | static int smp_pSeries_cpu_bootable(unsigned int nr) | |
191 | { | |
192 | /* Special case - we inhibit secondary thread startup | |
6a75a6b8 | 193 | * during boot if the user requests it. |
1da177e4 | 194 | */ |
954e6da5 NF |
195 | if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) { |
196 | if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0) | |
197 | return 0; | |
198 | if (smt_enabled_at_boot | |
199 | && cpu_thread_in_core(nr) >= smt_enabled_at_boot) | |
200 | return 0; | |
201 | } | |
1da177e4 LT |
202 | |
203 | return 1; | |
204 | } | |
0b05ac6e | 205 | |
e5e84f0a IM |
206 | /* Only used on systems that support multiple IPI mechanisms */ |
207 | static void pSeries_cause_ipi_mux(int cpu, unsigned long data) | |
208 | { | |
209 | if (cpumask_test_cpu(cpu, cpu_sibling_mask(smp_processor_id()))) | |
210 | doorbell_cause_ipi(cpu, data); | |
211 | else | |
212 | xics_cause_ipi(cpu, data); | |
213 | } | |
214 | ||
215 | static __init int pSeries_smp_probe(void) | |
216 | { | |
217 | int ret = xics_smp_probe(); | |
218 | ||
219 | if (cpu_has_feature(CPU_FTR_DBELL)) { | |
220 | xics_cause_ipi = smp_ops->cause_ipi; | |
221 | smp_ops->cause_ipi = pSeries_cause_ipi_mux; | |
222 | } | |
223 | ||
224 | return ret; | |
225 | } | |
226 | ||
1da177e4 LT |
227 | static struct smp_ops_t pSeries_mpic_smp_ops = { |
228 | .message_pass = smp_mpic_message_pass, | |
229 | .probe = smp_mpic_probe, | |
230 | .kick_cpu = smp_pSeries_kick_cpu, | |
231 | .setup_cpu = smp_mpic_setup_cpu, | |
232 | }; | |
0b05ac6e | 233 | |
1da177e4 | 234 | static struct smp_ops_t pSeries_xics_smp_ops = { |
9ca980dc | 235 | .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */ |
e5e84f0a IM |
236 | .cause_ipi = NULL, /* Filled at runtime by pSeries_smp_probe() */ |
237 | .probe = pSeries_smp_probe, | |
1da177e4 LT |
238 | .kick_cpu = smp_pSeries_kick_cpu, |
239 | .setup_cpu = smp_xics_setup_cpu, | |
240 | .cpu_bootable = smp_pSeries_cpu_bootable, | |
241 | }; | |
242 | ||
243 | /* This is called very early */ | |
0ebfff14 | 244 | static void __init smp_init_pseries(void) |
1da177e4 LT |
245 | { |
246 | int i; | |
247 | ||
f7ebf352 | 248 | pr_debug(" -> smp_init_pSeries()\n"); |
1da177e4 | 249 | |
af831e1e AB |
250 | alloc_bootmem_cpumask_var(&of_spin_mask); |
251 | ||
1da177e4 | 252 | /* Mark threads which are still spinning in hold loops. */ |
0231c290 AB |
253 | if (cpu_has_feature(CPU_FTR_SMT)) { |
254 | for_each_present_cpu(i) { | |
6a75a6b8 | 255 | if (cpu_thread_in_core(i) == 0) |
af831e1e | 256 | cpumask_set_cpu(i, of_spin_mask); |
1da177e4 | 257 | } |
0231c290 | 258 | } else { |
af831e1e | 259 | cpumask_copy(of_spin_mask, cpu_present_mask); |
0231c290 | 260 | } |
1da177e4 | 261 | |
af831e1e | 262 | cpumask_clear_cpu(boot_cpuid, of_spin_mask); |
1da177e4 LT |
263 | |
264 | /* Non-lpar has additional take/give timebase */ | |
265 | if (rtas_token("freeze-time-base") != RTAS_UNKNOWN_SERVICE) { | |
c4007a2f BH |
266 | smp_ops->give_timebase = rtas_give_timebase; |
267 | smp_ops->take_timebase = rtas_take_timebase; | |
1da177e4 LT |
268 | } |
269 | ||
f7ebf352 | 270 | pr_debug(" <- smp_init_pSeries()\n"); |
1da177e4 LT |
271 | } |
272 | ||
0ebfff14 BH |
273 | void __init smp_init_pseries_mpic(void) |
274 | { | |
275 | smp_ops = &pSeries_mpic_smp_ops; | |
276 | ||
277 | smp_init_pseries(); | |
278 | } | |
0ebfff14 BH |
279 | |
280 | void __init smp_init_pseries_xics(void) | |
281 | { | |
282 | smp_ops = &pSeries_xics_smp_ops; | |
283 | ||
284 | smp_init_pseries(); | |
285 | } |