Commit | Line | Data |
---|---|---|
43eab878 WD |
1 | /* |
2 | * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. | |
3 | * | |
4 | * ARMv7 support: Jean Pihet <jpihet@mvista.com> | |
5 | * 2010 (c) MontaVista Software, LLC. | |
6 | * | |
7 | * Copied from ARMv6 code, with the low level code inspired | |
8 | * by the ARMv7 Oprofile code. | |
9 | * | |
10 | * Cortex-A8 has up to 4 configurable performance counters and | |
11 | * a single cycle counter. | |
12 | * Cortex-A9 has up to 31 configurable performance counters and | |
13 | * a single cycle counter. | |
14 | * | |
15 | * All counters can be enabled/disabled and IRQ masked separately. The cycle | |
16 | * counter and all 4 performance counters together can be reset separately. | |
17 | */ | |
18 | ||
19 | #ifdef CONFIG_CPU_V7 | |
20 | /* Common ARMv7 event types */ | |
21 | enum armv7_perf_types { | |
22 | ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, | |
23 | ARMV7_PERFCTR_IFETCH_MISS = 0x01, | |
24 | ARMV7_PERFCTR_ITLB_MISS = 0x02, | |
25 | ARMV7_PERFCTR_DCACHE_REFILL = 0x03, | |
26 | ARMV7_PERFCTR_DCACHE_ACCESS = 0x04, | |
27 | ARMV7_PERFCTR_DTLB_REFILL = 0x05, | |
28 | ARMV7_PERFCTR_DREAD = 0x06, | |
29 | ARMV7_PERFCTR_DWRITE = 0x07, | |
30 | ||
31 | ARMV7_PERFCTR_EXC_TAKEN = 0x09, | |
32 | ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, | |
33 | ARMV7_PERFCTR_CID_WRITE = 0x0B, | |
34 | /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. | |
35 | * It counts: | |
36 | * - all branch instructions, | |
37 | * - instructions that explicitly write the PC, | |
38 | * - exception generating instructions. | |
39 | */ | |
40 | ARMV7_PERFCTR_PC_WRITE = 0x0C, | |
41 | ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, | |
42 | ARMV7_PERFCTR_UNALIGNED_ACCESS = 0x0F, | |
43 | ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, | |
44 | ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, | |
45 | ||
46 | ARMV7_PERFCTR_PC_BRANCH_MIS_USED = 0x12, | |
47 | ||
48 | ARMV7_PERFCTR_CPU_CYCLES = 0xFF | |
49 | }; | |
50 | ||
51 | /* ARMv7 Cortex-A8 specific event types */ | |
52 | enum armv7_a8_perf_types { | |
53 | ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, | |
54 | ||
55 | ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, | |
56 | ||
57 | ARMV7_PERFCTR_WRITE_BUFFER_FULL = 0x40, | |
58 | ARMV7_PERFCTR_L2_STORE_MERGED = 0x41, | |
59 | ARMV7_PERFCTR_L2_STORE_BUFF = 0x42, | |
60 | ARMV7_PERFCTR_L2_ACCESS = 0x43, | |
61 | ARMV7_PERFCTR_L2_CACH_MISS = 0x44, | |
62 | ARMV7_PERFCTR_AXI_READ_CYCLES = 0x45, | |
63 | ARMV7_PERFCTR_AXI_WRITE_CYCLES = 0x46, | |
64 | ARMV7_PERFCTR_MEMORY_REPLAY = 0x47, | |
65 | ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY = 0x48, | |
66 | ARMV7_PERFCTR_L1_DATA_MISS = 0x49, | |
67 | ARMV7_PERFCTR_L1_INST_MISS = 0x4A, | |
68 | ARMV7_PERFCTR_L1_DATA_COLORING = 0x4B, | |
69 | ARMV7_PERFCTR_L1_NEON_DATA = 0x4C, | |
70 | ARMV7_PERFCTR_L1_NEON_CACH_DATA = 0x4D, | |
71 | ARMV7_PERFCTR_L2_NEON = 0x4E, | |
72 | ARMV7_PERFCTR_L2_NEON_HIT = 0x4F, | |
73 | ARMV7_PERFCTR_L1_INST = 0x50, | |
74 | ARMV7_PERFCTR_PC_RETURN_MIS_PRED = 0x51, | |
75 | ARMV7_PERFCTR_PC_BRANCH_FAILED = 0x52, | |
76 | ARMV7_PERFCTR_PC_BRANCH_TAKEN = 0x53, | |
77 | ARMV7_PERFCTR_PC_BRANCH_EXECUTED = 0x54, | |
78 | ARMV7_PERFCTR_OP_EXECUTED = 0x55, | |
79 | ARMV7_PERFCTR_CYCLES_INST_STALL = 0x56, | |
80 | ARMV7_PERFCTR_CYCLES_INST = 0x57, | |
81 | ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL = 0x58, | |
82 | ARMV7_PERFCTR_CYCLES_NEON_INST_STALL = 0x59, | |
83 | ARMV7_PERFCTR_NEON_CYCLES = 0x5A, | |
84 | ||
85 | ARMV7_PERFCTR_PMU0_EVENTS = 0x70, | |
86 | ARMV7_PERFCTR_PMU1_EVENTS = 0x71, | |
87 | ARMV7_PERFCTR_PMU_EVENTS = 0x72, | |
88 | }; | |
89 | ||
90 | /* ARMv7 Cortex-A9 specific event types */ | |
91 | enum armv7_a9_perf_types { | |
92 | ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC = 0x40, | |
93 | ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC = 0x41, | |
94 | ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC = 0x42, | |
95 | ||
96 | ARMV7_PERFCTR_COHERENT_LINE_MISS = 0x50, | |
97 | ARMV7_PERFCTR_COHERENT_LINE_HIT = 0x51, | |
98 | ||
99 | ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES = 0x60, | |
100 | ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES = 0x61, | |
101 | ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62, | |
102 | ARMV7_PERFCTR_STREX_EXECUTED_PASSED = 0x63, | |
103 | ARMV7_PERFCTR_STREX_EXECUTED_FAILED = 0x64, | |
104 | ARMV7_PERFCTR_DATA_EVICTION = 0x65, | |
105 | ARMV7_PERFCTR_ISSUE_STAGE_NO_INST = 0x66, | |
106 | ARMV7_PERFCTR_ISSUE_STAGE_EMPTY = 0x67, | |
107 | ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE = 0x68, | |
108 | ||
109 | ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E, | |
110 | ||
111 | ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST = 0x70, | |
112 | ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71, | |
113 | ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST = 0x72, | |
114 | ARMV7_PERFCTR_FP_EXECUTED_INST = 0x73, | |
115 | ARMV7_PERFCTR_NEON_EXECUTED_INST = 0x74, | |
116 | ||
117 | ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80, | |
118 | ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES = 0x81, | |
119 | ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES = 0x82, | |
120 | ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES = 0x83, | |
121 | ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES = 0x84, | |
122 | ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES = 0x85, | |
123 | ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES = 0x86, | |
124 | ||
125 | ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES = 0x8A, | |
126 | ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B, | |
127 | ||
128 | ARMV7_PERFCTR_ISB_INST = 0x90, | |
129 | ARMV7_PERFCTR_DSB_INST = 0x91, | |
130 | ARMV7_PERFCTR_DMB_INST = 0x92, | |
131 | ARMV7_PERFCTR_EXT_INTERRUPTS = 0x93, | |
132 | ||
133 | ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED = 0xA0, | |
134 | ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED = 0xA1, | |
135 | ARMV7_PERFCTR_PLE_FIFO_FLUSH = 0xA2, | |
136 | ARMV7_PERFCTR_PLE_RQST_COMPLETED = 0xA3, | |
137 | ARMV7_PERFCTR_PLE_FIFO_OVERFLOW = 0xA4, | |
138 | ARMV7_PERFCTR_PLE_RQST_PROG = 0xA5 | |
139 | }; | |
140 | ||
141 | /* | |
142 | * Cortex-A8 HW events mapping | |
143 | * | |
144 | * The hardware events that we support. We do support cache operations but | |
145 | * we have harvard caches and no way to combine instruction and data | |
146 | * accesses/misses in hardware. | |
147 | */ | |
148 | static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { | |
149 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, | |
150 | [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, | |
151 | [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, | |
152 | [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, | |
153 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, | |
154 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, | |
155 | [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, | |
156 | }; | |
157 | ||
158 | static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | |
159 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
160 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
161 | [C(L1D)] = { | |
162 | /* | |
163 | * The performance counters don't differentiate between read | |
164 | * and write accesses/misses so this isn't strictly correct, | |
165 | * but it's the best we can do. Writes and reads get | |
166 | * combined. | |
167 | */ | |
168 | [C(OP_READ)] = { | |
169 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, | |
170 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, | |
171 | }, | |
172 | [C(OP_WRITE)] = { | |
173 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, | |
174 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, | |
175 | }, | |
176 | [C(OP_PREFETCH)] = { | |
177 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
178 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
179 | }, | |
180 | }, | |
181 | [C(L1I)] = { | |
182 | [C(OP_READ)] = { | |
183 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, | |
184 | [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, | |
185 | }, | |
186 | [C(OP_WRITE)] = { | |
187 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, | |
188 | [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, | |
189 | }, | |
190 | [C(OP_PREFETCH)] = { | |
191 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
192 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
193 | }, | |
194 | }, | |
195 | [C(LL)] = { | |
196 | [C(OP_READ)] = { | |
197 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, | |
198 | [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, | |
199 | }, | |
200 | [C(OP_WRITE)] = { | |
201 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, | |
202 | [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, | |
203 | }, | |
204 | [C(OP_PREFETCH)] = { | |
205 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
206 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
207 | }, | |
208 | }, | |
209 | [C(DTLB)] = { | |
210 | /* | |
211 | * Only ITLB misses and DTLB refills are supported. | |
212 | * If users want the DTLB refills misses a raw counter | |
213 | * must be used. | |
214 | */ | |
215 | [C(OP_READ)] = { | |
216 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
217 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, | |
218 | }, | |
219 | [C(OP_WRITE)] = { | |
220 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
221 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, | |
222 | }, | |
223 | [C(OP_PREFETCH)] = { | |
224 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
225 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
226 | }, | |
227 | }, | |
228 | [C(ITLB)] = { | |
229 | [C(OP_READ)] = { | |
230 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
231 | [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, | |
232 | }, | |
233 | [C(OP_WRITE)] = { | |
234 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
235 | [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, | |
236 | }, | |
237 | [C(OP_PREFETCH)] = { | |
238 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
239 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
240 | }, | |
241 | }, | |
242 | [C(BPU)] = { | |
243 | [C(OP_READ)] = { | |
244 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, | |
245 | [C(RESULT_MISS)] | |
246 | = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, | |
247 | }, | |
248 | [C(OP_WRITE)] = { | |
249 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, | |
250 | [C(RESULT_MISS)] | |
251 | = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, | |
252 | }, | |
253 | [C(OP_PREFETCH)] = { | |
254 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
255 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
256 | }, | |
257 | }, | |
258 | }; | |
259 | ||
260 | /* | |
261 | * Cortex-A9 HW events mapping | |
262 | */ | |
263 | static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { | |
264 | [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, | |
265 | [PERF_COUNT_HW_INSTRUCTIONS] = | |
266 | ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE, | |
267 | [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_COHERENT_LINE_HIT, | |
268 | [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_COHERENT_LINE_MISS, | |
269 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, | |
270 | [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, | |
271 | [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, | |
272 | }; | |
273 | ||
274 | static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] | |
275 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
276 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
277 | [C(L1D)] = { | |
278 | /* | |
279 | * The performance counters don't differentiate between read | |
280 | * and write accesses/misses so this isn't strictly correct, | |
281 | * but it's the best we can do. Writes and reads get | |
282 | * combined. | |
283 | */ | |
284 | [C(OP_READ)] = { | |
285 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, | |
286 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, | |
287 | }, | |
288 | [C(OP_WRITE)] = { | |
289 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, | |
290 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, | |
291 | }, | |
292 | [C(OP_PREFETCH)] = { | |
293 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
294 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
295 | }, | |
296 | }, | |
297 | [C(L1I)] = { | |
298 | [C(OP_READ)] = { | |
299 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
300 | [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, | |
301 | }, | |
302 | [C(OP_WRITE)] = { | |
303 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
304 | [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, | |
305 | }, | |
306 | [C(OP_PREFETCH)] = { | |
307 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
308 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
309 | }, | |
310 | }, | |
311 | [C(LL)] = { | |
312 | [C(OP_READ)] = { | |
313 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
314 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
315 | }, | |
316 | [C(OP_WRITE)] = { | |
317 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
318 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
319 | }, | |
320 | [C(OP_PREFETCH)] = { | |
321 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
322 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
323 | }, | |
324 | }, | |
325 | [C(DTLB)] = { | |
326 | /* | |
327 | * Only ITLB misses and DTLB refills are supported. | |
328 | * If users want the DTLB refills misses a raw counter | |
329 | * must be used. | |
330 | */ | |
331 | [C(OP_READ)] = { | |
332 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
333 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, | |
334 | }, | |
335 | [C(OP_WRITE)] = { | |
336 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
337 | [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, | |
338 | }, | |
339 | [C(OP_PREFETCH)] = { | |
340 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
341 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
342 | }, | |
343 | }, | |
344 | [C(ITLB)] = { | |
345 | [C(OP_READ)] = { | |
346 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
347 | [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, | |
348 | }, | |
349 | [C(OP_WRITE)] = { | |
350 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
351 | [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, | |
352 | }, | |
353 | [C(OP_PREFETCH)] = { | |
354 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
355 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
356 | }, | |
357 | }, | |
358 | [C(BPU)] = { | |
359 | [C(OP_READ)] = { | |
360 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, | |
361 | [C(RESULT_MISS)] | |
362 | = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, | |
363 | }, | |
364 | [C(OP_WRITE)] = { | |
365 | [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, | |
366 | [C(RESULT_MISS)] | |
367 | = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, | |
368 | }, | |
369 | [C(OP_PREFETCH)] = { | |
370 | [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, | |
371 | [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, | |
372 | }, | |
373 | }, | |
374 | }; | |
375 | ||
376 | /* | |
377 | * Perf Events counters | |
378 | */ | |
379 | enum armv7_counters { | |
380 | ARMV7_CYCLE_COUNTER = 1, /* Cycle counter */ | |
381 | ARMV7_COUNTER0 = 2, /* First event counter */ | |
382 | }; | |
383 | ||
384 | /* | |
385 | * The cycle counter is ARMV7_CYCLE_COUNTER. | |
386 | * The first event counter is ARMV7_COUNTER0. | |
387 | * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1). | |
388 | */ | |
389 | #define ARMV7_COUNTER_LAST (ARMV7_COUNTER0 + armpmu->num_events - 1) | |
390 | ||
391 | /* | |
392 | * ARMv7 low level PMNC access | |
393 | */ | |
394 | ||
395 | /* | |
396 | * Per-CPU PMNC: config reg | |
397 | */ | |
398 | #define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ | |
399 | #define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ | |
400 | #define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ | |
401 | #define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ | |
402 | #define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ | |
403 | #define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ | |
404 | #define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ | |
405 | #define ARMV7_PMNC_N_MASK 0x1f | |
406 | #define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ | |
407 | ||
408 | /* | |
409 | * Available counters | |
410 | */ | |
411 | #define ARMV7_CNT0 0 /* First event counter */ | |
412 | #define ARMV7_CCNT 31 /* Cycle counter */ | |
413 | ||
414 | /* Perf Event to low level counters mapping */ | |
415 | #define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0) | |
416 | ||
417 | /* | |
418 | * CNTENS: counters enable reg | |
419 | */ | |
420 | #define ARMV7_CNTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) | |
421 | #define ARMV7_CNTENS_C (1 << ARMV7_CCNT) | |
422 | ||
423 | /* | |
424 | * CNTENC: counters disable reg | |
425 | */ | |
426 | #define ARMV7_CNTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) | |
427 | #define ARMV7_CNTENC_C (1 << ARMV7_CCNT) | |
428 | ||
429 | /* | |
430 | * INTENS: counters overflow interrupt enable reg | |
431 | */ | |
432 | #define ARMV7_INTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) | |
433 | #define ARMV7_INTENS_C (1 << ARMV7_CCNT) | |
434 | ||
435 | /* | |
436 | * INTENC: counters overflow interrupt disable reg | |
437 | */ | |
438 | #define ARMV7_INTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) | |
439 | #define ARMV7_INTENC_C (1 << ARMV7_CCNT) | |
440 | ||
441 | /* | |
442 | * EVTSEL: Event selection reg | |
443 | */ | |
444 | #define ARMV7_EVTSEL_MASK 0xff /* Mask for writable bits */ | |
445 | ||
446 | /* | |
447 | * SELECT: Counter selection reg | |
448 | */ | |
449 | #define ARMV7_SELECT_MASK 0x1f /* Mask for writable bits */ | |
450 | ||
451 | /* | |
452 | * FLAG: counters overflow flag status reg | |
453 | */ | |
454 | #define ARMV7_FLAG_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) | |
455 | #define ARMV7_FLAG_C (1 << ARMV7_CCNT) | |
456 | #define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ | |
457 | #define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK | |
458 | ||
459 | static inline unsigned long armv7_pmnc_read(void) | |
460 | { | |
461 | u32 val; | |
462 | asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); | |
463 | return val; | |
464 | } | |
465 | ||
466 | static inline void armv7_pmnc_write(unsigned long val) | |
467 | { | |
468 | val &= ARMV7_PMNC_MASK; | |
469 | asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); | |
470 | } | |
471 | ||
472 | static inline int armv7_pmnc_has_overflowed(unsigned long pmnc) | |
473 | { | |
474 | return pmnc & ARMV7_OVERFLOWED_MASK; | |
475 | } | |
476 | ||
477 | static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc, | |
478 | enum armv7_counters counter) | |
479 | { | |
480 | int ret = 0; | |
481 | ||
482 | if (counter == ARMV7_CYCLE_COUNTER) | |
483 | ret = pmnc & ARMV7_FLAG_C; | |
484 | else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST)) | |
485 | ret = pmnc & ARMV7_FLAG_P(counter); | |
486 | else | |
487 | pr_err("CPU%u checking wrong counter %d overflow status\n", | |
488 | smp_processor_id(), counter); | |
489 | ||
490 | return ret; | |
491 | } | |
492 | ||
493 | static inline int armv7_pmnc_select_counter(unsigned int idx) | |
494 | { | |
495 | u32 val; | |
496 | ||
497 | if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) { | |
498 | pr_err("CPU%u selecting wrong PMNC counter" | |
499 | " %d\n", smp_processor_id(), idx); | |
500 | return -1; | |
501 | } | |
502 | ||
503 | val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK; | |
504 | asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); | |
505 | ||
506 | return idx; | |
507 | } | |
508 | ||
509 | static inline u32 armv7pmu_read_counter(int idx) | |
510 | { | |
511 | unsigned long value = 0; | |
512 | ||
513 | if (idx == ARMV7_CYCLE_COUNTER) | |
514 | asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); | |
515 | else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { | |
516 | if (armv7_pmnc_select_counter(idx) == idx) | |
517 | asm volatile("mrc p15, 0, %0, c9, c13, 2" | |
518 | : "=r" (value)); | |
519 | } else | |
520 | pr_err("CPU%u reading wrong counter %d\n", | |
521 | smp_processor_id(), idx); | |
522 | ||
523 | return value; | |
524 | } | |
525 | ||
526 | static inline void armv7pmu_write_counter(int idx, u32 value) | |
527 | { | |
528 | if (idx == ARMV7_CYCLE_COUNTER) | |
529 | asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); | |
530 | else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { | |
531 | if (armv7_pmnc_select_counter(idx) == idx) | |
532 | asm volatile("mcr p15, 0, %0, c9, c13, 2" | |
533 | : : "r" (value)); | |
534 | } else | |
535 | pr_err("CPU%u writing wrong counter %d\n", | |
536 | smp_processor_id(), idx); | |
537 | } | |
538 | ||
539 | static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val) | |
540 | { | |
541 | if (armv7_pmnc_select_counter(idx) == idx) { | |
542 | val &= ARMV7_EVTSEL_MASK; | |
543 | asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); | |
544 | } | |
545 | } | |
546 | ||
547 | static inline u32 armv7_pmnc_enable_counter(unsigned int idx) | |
548 | { | |
549 | u32 val; | |
550 | ||
551 | if ((idx != ARMV7_CYCLE_COUNTER) && | |
552 | ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { | |
553 | pr_err("CPU%u enabling wrong PMNC counter" | |
554 | " %d\n", smp_processor_id(), idx); | |
555 | return -1; | |
556 | } | |
557 | ||
558 | if (idx == ARMV7_CYCLE_COUNTER) | |
559 | val = ARMV7_CNTENS_C; | |
560 | else | |
561 | val = ARMV7_CNTENS_P(idx); | |
562 | ||
563 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); | |
564 | ||
565 | return idx; | |
566 | } | |
567 | ||
568 | static inline u32 armv7_pmnc_disable_counter(unsigned int idx) | |
569 | { | |
570 | u32 val; | |
571 | ||
572 | ||
573 | if ((idx != ARMV7_CYCLE_COUNTER) && | |
574 | ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { | |
575 | pr_err("CPU%u disabling wrong PMNC counter" | |
576 | " %d\n", smp_processor_id(), idx); | |
577 | return -1; | |
578 | } | |
579 | ||
580 | if (idx == ARMV7_CYCLE_COUNTER) | |
581 | val = ARMV7_CNTENC_C; | |
582 | else | |
583 | val = ARMV7_CNTENC_P(idx); | |
584 | ||
585 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); | |
586 | ||
587 | return idx; | |
588 | } | |
589 | ||
590 | static inline u32 armv7_pmnc_enable_intens(unsigned int idx) | |
591 | { | |
592 | u32 val; | |
593 | ||
594 | if ((idx != ARMV7_CYCLE_COUNTER) && | |
595 | ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { | |
596 | pr_err("CPU%u enabling wrong PMNC counter" | |
597 | " interrupt enable %d\n", smp_processor_id(), idx); | |
598 | return -1; | |
599 | } | |
600 | ||
601 | if (idx == ARMV7_CYCLE_COUNTER) | |
602 | val = ARMV7_INTENS_C; | |
603 | else | |
604 | val = ARMV7_INTENS_P(idx); | |
605 | ||
606 | asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); | |
607 | ||
608 | return idx; | |
609 | } | |
610 | ||
611 | static inline u32 armv7_pmnc_disable_intens(unsigned int idx) | |
612 | { | |
613 | u32 val; | |
614 | ||
615 | if ((idx != ARMV7_CYCLE_COUNTER) && | |
616 | ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { | |
617 | pr_err("CPU%u disabling wrong PMNC counter" | |
618 | " interrupt enable %d\n", smp_processor_id(), idx); | |
619 | return -1; | |
620 | } | |
621 | ||
622 | if (idx == ARMV7_CYCLE_COUNTER) | |
623 | val = ARMV7_INTENC_C; | |
624 | else | |
625 | val = ARMV7_INTENC_P(idx); | |
626 | ||
627 | asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); | |
628 | ||
629 | return idx; | |
630 | } | |
631 | ||
632 | static inline u32 armv7_pmnc_getreset_flags(void) | |
633 | { | |
634 | u32 val; | |
635 | ||
636 | /* Read */ | |
637 | asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); | |
638 | ||
639 | /* Write to clear flags */ | |
640 | val &= ARMV7_FLAG_MASK; | |
641 | asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); | |
642 | ||
643 | return val; | |
644 | } | |
645 | ||
646 | #ifdef DEBUG | |
647 | static void armv7_pmnc_dump_regs(void) | |
648 | { | |
649 | u32 val; | |
650 | unsigned int cnt; | |
651 | ||
652 | printk(KERN_INFO "PMNC registers dump:\n"); | |
653 | ||
654 | asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); | |
655 | printk(KERN_INFO "PMNC =0x%08x\n", val); | |
656 | ||
657 | asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); | |
658 | printk(KERN_INFO "CNTENS=0x%08x\n", val); | |
659 | ||
660 | asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); | |
661 | printk(KERN_INFO "INTENS=0x%08x\n", val); | |
662 | ||
663 | asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); | |
664 | printk(KERN_INFO "FLAGS =0x%08x\n", val); | |
665 | ||
666 | asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); | |
667 | printk(KERN_INFO "SELECT=0x%08x\n", val); | |
668 | ||
669 | asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); | |
670 | printk(KERN_INFO "CCNT =0x%08x\n", val); | |
671 | ||
672 | for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) { | |
673 | armv7_pmnc_select_counter(cnt); | |
674 | asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); | |
675 | printk(KERN_INFO "CNT[%d] count =0x%08x\n", | |
676 | cnt-ARMV7_EVENT_CNT_TO_CNTx, val); | |
677 | asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); | |
678 | printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n", | |
679 | cnt-ARMV7_EVENT_CNT_TO_CNTx, val); | |
680 | } | |
681 | } | |
682 | #endif | |
683 | ||
4d6b7a77 | 684 | static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx) |
43eab878 WD |
685 | { |
686 | unsigned long flags; | |
687 | ||
688 | /* | |
689 | * Enable counter and interrupt, and set the counter to count | |
690 | * the event that we're interested in. | |
691 | */ | |
961ec6da | 692 | raw_spin_lock_irqsave(&pmu_lock, flags); |
43eab878 WD |
693 | |
694 | /* | |
695 | * Disable counter | |
696 | */ | |
697 | armv7_pmnc_disable_counter(idx); | |
698 | ||
699 | /* | |
700 | * Set event (if destined for PMNx counters) | |
701 | * We don't need to set the event if it's a cycle count | |
702 | */ | |
703 | if (idx != ARMV7_CYCLE_COUNTER) | |
704 | armv7_pmnc_write_evtsel(idx, hwc->config_base); | |
705 | ||
706 | /* | |
707 | * Enable interrupt for this counter | |
708 | */ | |
709 | armv7_pmnc_enable_intens(idx); | |
710 | ||
711 | /* | |
712 | * Enable counter | |
713 | */ | |
714 | armv7_pmnc_enable_counter(idx); | |
715 | ||
961ec6da | 716 | raw_spin_unlock_irqrestore(&pmu_lock, flags); |
43eab878 WD |
717 | } |
718 | ||
719 | static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx) | |
720 | { | |
721 | unsigned long flags; | |
722 | ||
723 | /* | |
724 | * Disable counter and interrupt | |
725 | */ | |
961ec6da | 726 | raw_spin_lock_irqsave(&pmu_lock, flags); |
43eab878 WD |
727 | |
728 | /* | |
729 | * Disable counter | |
730 | */ | |
731 | armv7_pmnc_disable_counter(idx); | |
732 | ||
733 | /* | |
734 | * Disable interrupt for this counter | |
735 | */ | |
736 | armv7_pmnc_disable_intens(idx); | |
737 | ||
961ec6da | 738 | raw_spin_unlock_irqrestore(&pmu_lock, flags); |
43eab878 WD |
739 | } |
740 | ||
741 | static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) | |
742 | { | |
743 | unsigned long pmnc; | |
744 | struct perf_sample_data data; | |
745 | struct cpu_hw_events *cpuc; | |
746 | struct pt_regs *regs; | |
747 | int idx; | |
748 | ||
749 | /* | |
750 | * Get and reset the IRQ flags | |
751 | */ | |
752 | pmnc = armv7_pmnc_getreset_flags(); | |
753 | ||
754 | /* | |
755 | * Did an overflow occur? | |
756 | */ | |
757 | if (!armv7_pmnc_has_overflowed(pmnc)) | |
758 | return IRQ_NONE; | |
759 | ||
760 | /* | |
761 | * Handle the counter(s) overflow(s) | |
762 | */ | |
763 | regs = get_irq_regs(); | |
764 | ||
765 | perf_sample_data_init(&data, 0); | |
766 | ||
767 | cpuc = &__get_cpu_var(cpu_hw_events); | |
768 | for (idx = 0; idx <= armpmu->num_events; ++idx) { | |
769 | struct perf_event *event = cpuc->events[idx]; | |
770 | struct hw_perf_event *hwc; | |
771 | ||
772 | if (!test_bit(idx, cpuc->active_mask)) | |
773 | continue; | |
774 | ||
775 | /* | |
776 | * We have a single interrupt for all counters. Check that | |
777 | * each counter has overflowed before we process it. | |
778 | */ | |
779 | if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) | |
780 | continue; | |
781 | ||
782 | hwc = &event->hw; | |
783 | armpmu_event_update(event, hwc, idx); | |
784 | data.period = event->hw.last_period; | |
785 | if (!armpmu_event_set_period(event, hwc, idx)) | |
786 | continue; | |
787 | ||
788 | if (perf_event_overflow(event, 0, &data, regs)) | |
789 | armpmu->disable(hwc, idx); | |
790 | } | |
791 | ||
792 | /* | |
793 | * Handle the pending perf events. | |
794 | * | |
795 | * Note: this call *must* be run with interrupts disabled. For | |
796 | * platforms that can have the PMU interrupts raised as an NMI, this | |
797 | * will not work. | |
798 | */ | |
799 | irq_work_run(); | |
800 | ||
801 | return IRQ_HANDLED; | |
802 | } | |
803 | ||
804 | static void armv7pmu_start(void) | |
805 | { | |
806 | unsigned long flags; | |
807 | ||
961ec6da | 808 | raw_spin_lock_irqsave(&pmu_lock, flags); |
43eab878 WD |
809 | /* Enable all counters */ |
810 | armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); | |
961ec6da | 811 | raw_spin_unlock_irqrestore(&pmu_lock, flags); |
43eab878 WD |
812 | } |
813 | ||
814 | static void armv7pmu_stop(void) | |
815 | { | |
816 | unsigned long flags; | |
817 | ||
961ec6da | 818 | raw_spin_lock_irqsave(&pmu_lock, flags); |
43eab878 WD |
819 | /* Disable all counters */ |
820 | armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); | |
961ec6da | 821 | raw_spin_unlock_irqrestore(&pmu_lock, flags); |
43eab878 WD |
822 | } |
823 | ||
824 | static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc, | |
825 | struct hw_perf_event *event) | |
826 | { | |
827 | int idx; | |
828 | ||
829 | /* Always place a cycle counter into the cycle counter. */ | |
830 | if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) { | |
831 | if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask)) | |
832 | return -EAGAIN; | |
833 | ||
834 | return ARMV7_CYCLE_COUNTER; | |
835 | } else { | |
836 | /* | |
837 | * For anything other than a cycle counter, try and use | |
838 | * the events counters | |
839 | */ | |
840 | for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) { | |
841 | if (!test_and_set_bit(idx, cpuc->used_mask)) | |
842 | return idx; | |
843 | } | |
844 | ||
845 | /* The counters are all in use. */ | |
846 | return -EAGAIN; | |
847 | } | |
848 | } | |
849 | ||
850 | static struct arm_pmu armv7pmu = { | |
851 | .handle_irq = armv7pmu_handle_irq, | |
852 | .enable = armv7pmu_enable_event, | |
853 | .disable = armv7pmu_disable_event, | |
854 | .read_counter = armv7pmu_read_counter, | |
855 | .write_counter = armv7pmu_write_counter, | |
856 | .get_event_idx = armv7pmu_get_event_idx, | |
857 | .start = armv7pmu_start, | |
858 | .stop = armv7pmu_stop, | |
859 | .raw_event_mask = 0xFF, | |
860 | .max_period = (1LLU << 32) - 1, | |
861 | }; | |
862 | ||
863 | static u32 __init armv7_reset_read_pmnc(void) | |
864 | { | |
865 | u32 nb_cnt; | |
866 | ||
867 | /* Initialize & Reset PMNC: C and P bits */ | |
868 | armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); | |
869 | ||
870 | /* Read the nb of CNTx counters supported from PMNC */ | |
871 | nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; | |
872 | ||
873 | /* Add the CPU cycles counter and return */ | |
874 | return nb_cnt + 1; | |
875 | } | |
876 | ||
4d6b7a77 | 877 | static const struct arm_pmu *__init armv7_a8_pmu_init(void) |
43eab878 WD |
878 | { |
879 | armv7pmu.id = ARM_PERF_PMU_ID_CA8; | |
880 | armv7pmu.name = "ARMv7 Cortex-A8"; | |
881 | armv7pmu.cache_map = &armv7_a8_perf_cache_map; | |
882 | armv7pmu.event_map = &armv7_a8_perf_map; | |
883 | armv7pmu.num_events = armv7_reset_read_pmnc(); | |
884 | return &armv7pmu; | |
885 | } | |
886 | ||
4d6b7a77 | 887 | static const struct arm_pmu *__init armv7_a9_pmu_init(void) |
43eab878 WD |
888 | { |
889 | armv7pmu.id = ARM_PERF_PMU_ID_CA9; | |
890 | armv7pmu.name = "ARMv7 Cortex-A9"; | |
891 | armv7pmu.cache_map = &armv7_a9_perf_cache_map; | |
892 | armv7pmu.event_map = &armv7_a9_perf_map; | |
893 | armv7pmu.num_events = armv7_reset_read_pmnc(); | |
894 | return &armv7pmu; | |
895 | } | |
896 | #else | |
4d6b7a77 | 897 | static const struct arm_pmu *__init armv7_a8_pmu_init(void) |
43eab878 WD |
898 | { |
899 | return NULL; | |
900 | } | |
901 | ||
4d6b7a77 | 902 | static const struct arm_pmu *__init armv7_a9_pmu_init(void) |
43eab878 WD |
903 | { |
904 | return NULL; | |
905 | } | |
906 | #endif /* CONFIG_CPU_V7 */ |