Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Copyright (C) 2004 Anton Blanchard <anton@au.ibm.com>, IBM | |
078f1940 | 3 | * Added mmcra[slot] support: |
4 | * Copyright (C) 2006-2007 Will Schmidt <willschm@us.ibm.com>, IBM | |
1da177e4 LT |
5 | * |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | #include <linux/oprofile.h> | |
13 | #include <linux/init.h> | |
14 | #include <linux/smp.h> | |
57cfb814 | 15 | #include <asm/firmware.h> |
1da177e4 | 16 | #include <asm/ptrace.h> |
1da177e4 LT |
17 | #include <asm/processor.h> |
18 | #include <asm/cputable.h> | |
1da177e4 | 19 | #include <asm/rtas.h> |
dca85932 | 20 | #include <asm/oprofile_impl.h> |
cb09cff3 | 21 | #include <asm/reg.h> |
1da177e4 LT |
22 | |
23 | #define dbg(args...) | |
adbf115d CL |
24 | #define OPROFILE_PM_PMCSEL_MSK 0xffULL |
25 | #define OPROFILE_PM_UNIT_SHIFT 60 | |
26 | #define OPROFILE_PM_UNIT_MSK 0xfULL | |
27 | #define OPROFILE_MAX_PMC_NUM 3 | |
28 | #define OPROFILE_PMSEL_FIELD_WIDTH 8 | |
29 | #define OPROFILE_UNIT_FIELD_WIDTH 4 | |
30 | #define MMCRA_SIAR_VALID_MASK 0x10000000ULL | |
1da177e4 | 31 | |
1da177e4 LT |
32 | static unsigned long reset_value[OP_MAX_COUNTER]; |
33 | ||
1da177e4 | 34 | static int oprofile_running; |
e5fc948b | 35 | static int use_slot_nums; |
1da177e4 LT |
36 | |
37 | /* mmcr values are set in power4_reg_setup, used in power4_cpu_setup */ | |
38 | static u32 mmcr0_val; | |
39 | static u64 mmcr1_val; | |
15e812ad | 40 | static u64 mmcra_val; |
adbf115d CL |
41 | static u32 cntr_marked_events; |
42 | ||
43 | static int power7_marked_instr_event(u64 mmcr1) | |
44 | { | |
45 | u64 psel, unit; | |
46 | int pmc, cntr_marked_events = 0; | |
47 | ||
48 | /* Given the MMCR1 value, look at the field for each counter to | |
49 | * determine if it is a marked event. Code based on the function | |
50 | * power7_marked_instr_event() in file arch/powerpc/perf/power7-pmu.c. | |
51 | */ | |
52 | for (pmc = 0; pmc < 4; pmc++) { | |
53 | psel = mmcr1 & (OPROFILE_PM_PMCSEL_MSK | |
54 | << (OPROFILE_MAX_PMC_NUM - pmc) | |
46ed7a76 | 55 | * OPROFILE_PMSEL_FIELD_WIDTH); |
adbf115d CL |
56 | psel = (psel >> ((OPROFILE_MAX_PMC_NUM - pmc) |
57 | * OPROFILE_PMSEL_FIELD_WIDTH)) & ~1ULL; | |
58 | unit = mmcr1 & (OPROFILE_PM_UNIT_MSK | |
59 | << (OPROFILE_PM_UNIT_SHIFT | |
60 | - (pmc * OPROFILE_PMSEL_FIELD_WIDTH ))); | |
61 | unit = unit >> (OPROFILE_PM_UNIT_SHIFT | |
62 | - (pmc * OPROFILE_PMSEL_FIELD_WIDTH)); | |
63 | ||
64 | switch (psel >> 4) { | |
65 | case 2: | |
66 | cntr_marked_events |= (pmc == 1 || pmc == 3) << pmc; | |
67 | break; | |
68 | case 3: | |
69 | if (psel == 0x3c) { | |
70 | cntr_marked_events |= (pmc == 0) << pmc; | |
71 | break; | |
72 | } | |
73 | ||
74 | if (psel == 0x3e) { | |
75 | cntr_marked_events |= (pmc != 1) << pmc; | |
76 | break; | |
77 | } | |
78 | ||
79 | cntr_marked_events |= 1 << pmc; | |
80 | break; | |
81 | case 4: | |
82 | case 5: | |
83 | cntr_marked_events |= (unit == 0xd) << pmc; | |
84 | break; | |
85 | case 6: | |
86 | if (psel == 0x64) | |
87 | cntr_marked_events |= (pmc >= 2) << pmc; | |
88 | break; | |
89 | case 8: | |
90 | cntr_marked_events |= (unit == 0xd) << pmc; | |
91 | break; | |
92 | } | |
93 | } | |
94 | return cntr_marked_events; | |
95 | } | |
1da177e4 | 96 | |
1474855d | 97 | static int power4_reg_setup(struct op_counter_config *ctr, |
1da177e4 LT |
98 | struct op_system_config *sys, |
99 | int num_ctrs) | |
100 | { | |
101 | int i; | |
102 | ||
1da177e4 LT |
103 | /* |
104 | * The performance counter event settings are given in the mmcr0, | |
105 | * mmcr1 and mmcra values passed from the user in the | |
106 | * op_system_config structure (sys variable). | |
107 | */ | |
108 | mmcr0_val = sys->mmcr0; | |
109 | mmcr1_val = sys->mmcr1; | |
110 | mmcra_val = sys->mmcra; | |
111 | ||
adbf115d CL |
112 | /* Power 7+ and newer architectures: |
113 | * Determine which counter events in the group (the group of events is | |
114 | * specified by the bit settings in the MMCR1 register) are marked | |
115 | * events for use in the interrupt handler. Do the calculation once | |
116 | * before OProfile starts. Information is used in the interrupt | |
117 | * handler. Starting with Power 7+ we only record the sample for | |
118 | * marked events if the SIAR valid bit is set. For non marked events | |
119 | * the sample is always recorded. | |
120 | */ | |
121 | if (pvr_version_is(PVR_POWER7p)) | |
122 | cntr_marked_events = power7_marked_instr_event(mmcr1_val); | |
123 | else | |
124 | cntr_marked_events = 0; /* For older processors, set the bit map | |
125 | * to zero so the sample will always be | |
126 | * be recorded. | |
127 | */ | |
128 | ||
a6908cd0 | 129 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) |
1da177e4 LT |
130 | reset_value[i] = 0x80000000UL - ctr[i].count; |
131 | ||
132 | /* setup user and kernel profiling */ | |
133 | if (sys->enable_kernel) | |
134 | mmcr0_val &= ~MMCR0_KERNEL_DISABLE; | |
135 | else | |
136 | mmcr0_val |= MMCR0_KERNEL_DISABLE; | |
137 | ||
138 | if (sys->enable_user) | |
139 | mmcr0_val &= ~MMCR0_PROBLEM_DISABLE; | |
140 | else | |
141 | mmcr0_val |= MMCR0_PROBLEM_DISABLE; | |
1474855d | 142 | |
d3dbeef6 ME |
143 | if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) || |
144 | pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) || | |
145 | pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX) || | |
146 | pvr_version_is(PVR_POWER5) || pvr_version_is(PVR_POWER5p)) | |
e5fc948b MJ |
147 | use_slot_nums = 1; |
148 | ||
1474855d | 149 | return 0; |
1da177e4 LT |
150 | } |
151 | ||
b950bdd0 | 152 | extern void ppc_enable_pmcs(void); |
1da177e4 | 153 | |
cb09cff3 AB |
154 | /* |
155 | * Older CPUs require the MMCRA sample bit to be always set, but newer | |
156 | * CPUs only want it set for some groups. Eventually we will remove all | |
157 | * knowledge of this bit in the kernel, oprofile userspace should be | |
158 | * setting it when required. | |
159 | * | |
160 | * In order to keep current installations working we force the bit for | |
161 | * those older CPUs. Once everyone has updated their oprofile userspace we | |
162 | * can remove this hack. | |
163 | */ | |
164 | static inline int mmcra_must_set_sample(void) | |
165 | { | |
d3dbeef6 ME |
166 | if (pvr_version_is(PVR_POWER4) || pvr_version_is(PVR_POWER4p) || |
167 | pvr_version_is(PVR_970) || pvr_version_is(PVR_970FX) || | |
168 | pvr_version_is(PVR_970MP) || pvr_version_is(PVR_970GX)) | |
cb09cff3 AB |
169 | return 1; |
170 | ||
171 | return 0; | |
172 | } | |
173 | ||
1474855d | 174 | static int power4_cpu_setup(struct op_counter_config *ctr) |
1da177e4 LT |
175 | { |
176 | unsigned int mmcr0 = mmcr0_val; | |
177 | unsigned long mmcra = mmcra_val; | |
178 | ||
b950bdd0 | 179 | ppc_enable_pmcs(); |
1da177e4 LT |
180 | |
181 | /* set the freeze bit */ | |
182 | mmcr0 |= MMCR0_FC; | |
183 | mtspr(SPRN_MMCR0, mmcr0); | |
184 | ||
185 | mmcr0 |= MMCR0_FCM1|MMCR0_PMXE|MMCR0_FCECE; | |
186 | mmcr0 |= MMCR0_PMC1CE|MMCR0_PMCjCE; | |
187 | mtspr(SPRN_MMCR0, mmcr0); | |
188 | ||
189 | mtspr(SPRN_MMCR1, mmcr1_val); | |
190 | ||
cb09cff3 AB |
191 | if (mmcra_must_set_sample()) |
192 | mmcra |= MMCRA_SAMPLE_ENABLE; | |
1da177e4 LT |
193 | mtspr(SPRN_MMCRA, mmcra); |
194 | ||
195 | dbg("setup on cpu %d, mmcr0 %lx\n", smp_processor_id(), | |
196 | mfspr(SPRN_MMCR0)); | |
197 | dbg("setup on cpu %d, mmcr1 %lx\n", smp_processor_id(), | |
198 | mfspr(SPRN_MMCR1)); | |
199 | dbg("setup on cpu %d, mmcra %lx\n", smp_processor_id(), | |
200 | mfspr(SPRN_MMCRA)); | |
1474855d BN |
201 | |
202 | return 0; | |
1da177e4 LT |
203 | } |
204 | ||
1474855d | 205 | static int power4_start(struct op_counter_config *ctr) |
1da177e4 LT |
206 | { |
207 | int i; | |
208 | unsigned int mmcr0; | |
209 | ||
210 | /* set the PMM bit (see comment below) */ | |
211 | mtmsrd(mfmsr() | MSR_PMM); | |
212 | ||
a6908cd0 | 213 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { |
1da177e4 | 214 | if (ctr[i].enabled) { |
c69b767a | 215 | classic_ctr_write(i, reset_value[i]); |
1da177e4 | 216 | } else { |
c69b767a | 217 | classic_ctr_write(i, 0); |
1da177e4 LT |
218 | } |
219 | } | |
220 | ||
221 | mmcr0 = mfspr(SPRN_MMCR0); | |
222 | ||
223 | /* | |
224 | * We must clear the PMAO bit on some (GQ) chips. Just do it | |
225 | * all the time | |
226 | */ | |
227 | mmcr0 &= ~MMCR0_PMAO; | |
228 | ||
229 | /* | |
230 | * now clear the freeze bit, counting will not start until we | |
231 | * rfid from this excetion, because only at that point will | |
232 | * the PMM bit be cleared | |
233 | */ | |
234 | mmcr0 &= ~MMCR0_FC; | |
235 | mtspr(SPRN_MMCR0, mmcr0); | |
236 | ||
237 | oprofile_running = 1; | |
238 | ||
239 | dbg("start on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0); | |
1474855d | 240 | return 0; |
1da177e4 LT |
241 | } |
242 | ||
243 | static void power4_stop(void) | |
244 | { | |
245 | unsigned int mmcr0; | |
246 | ||
247 | /* freeze counters */ | |
248 | mmcr0 = mfspr(SPRN_MMCR0); | |
249 | mmcr0 |= MMCR0_FC; | |
250 | mtspr(SPRN_MMCR0, mmcr0); | |
251 | ||
252 | oprofile_running = 0; | |
253 | ||
254 | dbg("stop on cpu %d, mmcr0 %x\n", smp_processor_id(), mmcr0); | |
255 | ||
256 | mb(); | |
257 | } | |
258 | ||
259 | /* Fake functions used by canonicalize_pc */ | |
3ff6eecc | 260 | static void __used hypervisor_bucket(void) |
1da177e4 LT |
261 | { |
262 | } | |
263 | ||
3ff6eecc | 264 | static void __used rtas_bucket(void) |
1da177e4 LT |
265 | { |
266 | } | |
267 | ||
3ff6eecc | 268 | static void __used kernel_unknown_bucket(void) |
1da177e4 LT |
269 | { |
270 | } | |
271 | ||
1da177e4 LT |
272 | /* |
273 | * On GQ and newer the MMCRA stores the HV and PR bits at the time | |
274 | * the SIAR was sampled. We use that to work out if the SIAR was sampled in | |
275 | * the hypervisor, our exception vectors or RTAS. | |
078f1940 | 276 | * If the MMCRA_SAMPLE_ENABLE bit is set, we can use the MMCRA[slot] bits |
277 | * to more accurately identify the address of the sampled instruction. The | |
278 | * mmcra[slot] bits represent the slot number of a sampled instruction | |
279 | * within an instruction group. The slot will contain a value between 1 | |
280 | * and 5 if MMCRA_SAMPLE_ENABLE is set, otherwise 0. | |
1da177e4 LT |
281 | */ |
282 | static unsigned long get_pc(struct pt_regs *regs) | |
283 | { | |
284 | unsigned long pc = mfspr(SPRN_SIAR); | |
285 | unsigned long mmcra; | |
078f1940 | 286 | unsigned long slot; |
1da177e4 | 287 | |
25985edc | 288 | /* Can't do much about it */ |
e78dbc80 | 289 | if (!cur_cpu_spec->oprofile_mmcra_sihv) |
15e812ad | 290 | return pc; |
1da177e4 LT |
291 | |
292 | mmcra = mfspr(SPRN_MMCRA); | |
293 | ||
e5fc948b | 294 | if (use_slot_nums && (mmcra & MMCRA_SAMPLE_ENABLE)) { |
078f1940 | 295 | slot = ((mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT); |
296 | if (slot > 1) | |
297 | pc += 4 * (slot - 1); | |
298 | } | |
299 | ||
1da177e4 | 300 | /* Were we in the hypervisor? */ |
e78dbc80 MN |
301 | if (firmware_has_feature(FW_FEATURE_LPAR) && |
302 | (mmcra & cur_cpu_spec->oprofile_mmcra_sihv)) | |
1da177e4 LT |
303 | /* function descriptor madness */ |
304 | return *((unsigned long *)hypervisor_bucket); | |
305 | ||
306 | /* We were in userspace, nothing to do */ | |
e78dbc80 | 307 | if (mmcra & cur_cpu_spec->oprofile_mmcra_sipr) |
1da177e4 LT |
308 | return pc; |
309 | ||
310 | #ifdef CONFIG_PPC_RTAS | |
311 | /* Were we in RTAS? */ | |
312 | if (pc >= rtas.base && pc < (rtas.base + rtas.size)) | |
313 | /* function descriptor madness */ | |
314 | return *((unsigned long *)rtas_bucket); | |
315 | #endif | |
316 | ||
317 | /* Were we in our exception vectors or SLB real mode miss handler? */ | |
318 | if (pc < 0x1000000UL) | |
319 | return (unsigned long)__va(pc); | |
320 | ||
321 | /* Not sure where we were */ | |
51fae6de | 322 | if (!is_kernel_addr(pc)) |
1da177e4 LT |
323 | /* function descriptor madness */ |
324 | return *((unsigned long *)kernel_unknown_bucket); | |
325 | ||
15e812ad | 326 | return pc; |
1da177e4 LT |
327 | } |
328 | ||
e78dbc80 | 329 | static int get_kernel(unsigned long pc, unsigned long mmcra) |
1da177e4 LT |
330 | { |
331 | int is_kernel; | |
332 | ||
e78dbc80 | 333 | if (!cur_cpu_spec->oprofile_mmcra_sihv) { |
51fae6de | 334 | is_kernel = is_kernel_addr(pc); |
1da177e4 | 335 | } else { |
e78dbc80 | 336 | is_kernel = ((mmcra & cur_cpu_spec->oprofile_mmcra_sipr) == 0); |
1da177e4 LT |
337 | } |
338 | ||
339 | return is_kernel; | |
340 | } | |
341 | ||
ad5d5292 EM |
342 | static bool pmc_overflow(unsigned long val) |
343 | { | |
344 | if ((int)val < 0) | |
345 | return true; | |
346 | ||
347 | /* | |
348 | * Events on POWER7 can roll back if a speculative event doesn't | |
349 | * eventually complete. Unfortunately in some rare cases they will | |
350 | * raise a performance monitor exception. We need to catch this to | |
351 | * ensure we reset the PMC. In all cases the PMC will be 256 or less | |
352 | * cycles from overflow. | |
353 | * | |
354 | * We only do this if the first pass fails to find any overflowing | |
355 | * PMCs because a user might set a period of less than 256 and we | |
356 | * don't want to mistakenly reset them. | |
357 | */ | |
d3dbeef6 | 358 | if (pvr_version_is(PVR_POWER7) && ((0x80000000 - val) <= 256)) |
ad5d5292 EM |
359 | return true; |
360 | ||
361 | return false; | |
362 | } | |
363 | ||
1da177e4 LT |
364 | static void power4_handle_interrupt(struct pt_regs *regs, |
365 | struct op_counter_config *ctr) | |
366 | { | |
367 | unsigned long pc; | |
368 | int is_kernel; | |
369 | int val; | |
370 | int i; | |
371 | unsigned int mmcr0; | |
e78dbc80 | 372 | unsigned long mmcra; |
adbf115d | 373 | bool siar_valid = false; |
e78dbc80 MN |
374 | |
375 | mmcra = mfspr(SPRN_MMCRA); | |
1da177e4 LT |
376 | |
377 | pc = get_pc(regs); | |
e78dbc80 | 378 | is_kernel = get_kernel(pc, mmcra); |
1da177e4 LT |
379 | |
380 | /* set the PMM bit (see comment below) */ | |
381 | mtmsrd(mfmsr() | MSR_PMM); | |
382 | ||
adbf115d CL |
383 | /* Check that the SIAR valid bit in MMCRA is set to 1. */ |
384 | if ((mmcra & MMCRA_SIAR_VALID_MASK) == MMCRA_SIAR_VALID_MASK) | |
385 | siar_valid = true; | |
386 | ||
a6908cd0 | 387 | for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { |
c69b767a | 388 | val = classic_ctr_read(i); |
ad5d5292 | 389 | if (pmc_overflow(val)) { |
1da177e4 | 390 | if (oprofile_running && ctr[i].enabled) { |
adbf115d CL |
391 | /* Power 7+ and newer architectures: |
392 | * If the event is a marked event, then only | |
393 | * save the sample if the SIAR valid bit is | |
394 | * set. If the event is not marked, then | |
395 | * always save the sample. | |
396 | * Note, the Sample enable bit in the MMCRA | |
397 | * register must be set to 1 if the group | |
398 | * contains a marked event. | |
399 | */ | |
400 | if ((siar_valid && | |
401 | (cntr_marked_events & (1 << i))) | |
402 | || !(cntr_marked_events & (1 << i))) | |
403 | oprofile_add_ext_sample(pc, regs, i, | |
404 | is_kernel); | |
405 | ||
c69b767a | 406 | classic_ctr_write(i, reset_value[i]); |
1da177e4 | 407 | } else { |
c69b767a | 408 | classic_ctr_write(i, 0); |
1da177e4 LT |
409 | } |
410 | } | |
411 | } | |
412 | ||
413 | mmcr0 = mfspr(SPRN_MMCR0); | |
414 | ||
415 | /* reset the perfmon trigger */ | |
416 | mmcr0 |= MMCR0_PMXE; | |
417 | ||
418 | /* | |
419 | * We must clear the PMAO bit on some (GQ) chips. Just do it | |
420 | * all the time | |
421 | */ | |
422 | mmcr0 &= ~MMCR0_PMAO; | |
423 | ||
e78dbc80 MN |
424 | /* Clear the appropriate bits in the MMCRA */ |
425 | mmcra &= ~cur_cpu_spec->oprofile_mmcra_clear; | |
426 | mtspr(SPRN_MMCRA, mmcra); | |
427 | ||
1da177e4 LT |
428 | /* |
429 | * now clear the freeze bit, counting will not start until we | |
430 | * rfid from this exception, because only at that point will | |
431 | * the PMM bit be cleared | |
432 | */ | |
433 | mmcr0 &= ~MMCR0_FC; | |
434 | mtspr(SPRN_MMCR0, mmcr0); | |
435 | } | |
436 | ||
a3e48c10 | 437 | struct op_powerpc_model op_model_power4 = { |
1da177e4 LT |
438 | .reg_setup = power4_reg_setup, |
439 | .cpu_setup = power4_cpu_setup, | |
440 | .start = power4_start, | |
441 | .stop = power4_stop, | |
442 | .handle_interrupt = power4_handle_interrupt, | |
443 | }; |