4 * Copyright (C) 2012 ARM Limited
5 * Author: Will Deacon <will.deacon@arm.com>
7 * This code is based heavily on the ARMv7 perf event code.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) "hw perfevents: " fmt
23 #include <linux/bitmap.h>
24 #include <linux/interrupt.h>
25 #include <linux/kernel.h>
26 #include <linux/export.h>
27 #include <linux/perf_event.h>
28 #include <linux/platform_device.h>
29 #include <linux/spinlock.h>
30 #include <linux/uaccess.h>
32 #include <asm/cputype.h>
34 #include <asm/irq_regs.h>
36 #include <asm/stacktrace.h>
39 * ARMv8 supports a maximum of 32 events.
40 * The cycle counter is included in this total.
42 #define ARMPMU_MAX_HWEVENTS 32
44 static DEFINE_PER_CPU(struct perf_event
* [ARMPMU_MAX_HWEVENTS
], hw_events
);
45 static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS
)], used_mask
);
46 static DEFINE_PER_CPU(struct pmu_hw_events
, cpu_hw_events
);
48 #define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
50 /* Set at runtime when we know what CPU type we are. */
51 static struct arm_pmu
*cpu_pmu
;
54 armpmu_get_max_events(void)
59 max_events
= cpu_pmu
->num_events
;
63 EXPORT_SYMBOL_GPL(armpmu_get_max_events
);
65 int perf_num_counters(void)
67 return armpmu_get_max_events();
69 EXPORT_SYMBOL_GPL(perf_num_counters
);
71 #define HW_OP_UNSUPPORTED 0xFFFF
74 PERF_COUNT_HW_CACHE_##_x
76 #define CACHE_OP_UNSUPPORTED 0xFFFF
79 armpmu_map_cache_event(const unsigned (*cache_map
)
80 [PERF_COUNT_HW_CACHE_MAX
]
81 [PERF_COUNT_HW_CACHE_OP_MAX
]
82 [PERF_COUNT_HW_CACHE_RESULT_MAX
],
85 unsigned int cache_type
, cache_op
, cache_result
, ret
;
87 cache_type
= (config
>> 0) & 0xff;
88 if (cache_type
>= PERF_COUNT_HW_CACHE_MAX
)
91 cache_op
= (config
>> 8) & 0xff;
92 if (cache_op
>= PERF_COUNT_HW_CACHE_OP_MAX
)
95 cache_result
= (config
>> 16) & 0xff;
96 if (cache_result
>= PERF_COUNT_HW_CACHE_RESULT_MAX
)
99 ret
= (int)(*cache_map
)[cache_type
][cache_op
][cache_result
];
101 if (ret
== CACHE_OP_UNSUPPORTED
)
108 armpmu_map_event(const unsigned (*event_map
)[PERF_COUNT_HW_MAX
], u64 config
)
112 if (config
>= PERF_COUNT_HW_MAX
)
115 mapping
= (*event_map
)[config
];
116 return mapping
== HW_OP_UNSUPPORTED
? -ENOENT
: mapping
;
120 armpmu_map_raw_event(u32 raw_event_mask
, u64 config
)
122 return (int)(config
& raw_event_mask
);
125 static int map_cpu_event(struct perf_event
*event
,
126 const unsigned (*event_map
)[PERF_COUNT_HW_MAX
],
127 const unsigned (*cache_map
)
128 [PERF_COUNT_HW_CACHE_MAX
]
129 [PERF_COUNT_HW_CACHE_OP_MAX
]
130 [PERF_COUNT_HW_CACHE_RESULT_MAX
],
133 u64 config
= event
->attr
.config
;
135 switch (event
->attr
.type
) {
136 case PERF_TYPE_HARDWARE
:
137 return armpmu_map_event(event_map
, config
);
138 case PERF_TYPE_HW_CACHE
:
139 return armpmu_map_cache_event(cache_map
, config
);
141 return armpmu_map_raw_event(raw_event_mask
, config
);
148 armpmu_event_set_period(struct perf_event
*event
,
149 struct hw_perf_event
*hwc
,
152 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
153 s64 left
= local64_read(&hwc
->period_left
);
154 s64 period
= hwc
->sample_period
;
157 if (unlikely(left
<= -period
)) {
159 local64_set(&hwc
->period_left
, left
);
160 hwc
->last_period
= period
;
164 if (unlikely(left
<= 0)) {
166 local64_set(&hwc
->period_left
, left
);
167 hwc
->last_period
= period
;
171 if (left
> (s64
)armpmu
->max_period
)
172 left
= armpmu
->max_period
;
174 local64_set(&hwc
->prev_count
, (u64
)-left
);
176 armpmu
->write_counter(idx
, (u64
)(-left
) & 0xffffffff);
178 perf_event_update_userpage(event
);
184 armpmu_event_update(struct perf_event
*event
,
185 struct hw_perf_event
*hwc
,
188 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
189 u64 delta
, prev_raw_count
, new_raw_count
;
192 prev_raw_count
= local64_read(&hwc
->prev_count
);
193 new_raw_count
= armpmu
->read_counter(idx
);
195 if (local64_cmpxchg(&hwc
->prev_count
, prev_raw_count
,
196 new_raw_count
) != prev_raw_count
)
199 delta
= (new_raw_count
- prev_raw_count
) & armpmu
->max_period
;
201 local64_add(delta
, &event
->count
);
202 local64_sub(delta
, &hwc
->period_left
);
204 return new_raw_count
;
208 armpmu_read(struct perf_event
*event
)
210 struct hw_perf_event
*hwc
= &event
->hw
;
212 /* Don't read disabled counters! */
216 armpmu_event_update(event
, hwc
, hwc
->idx
);
220 armpmu_stop(struct perf_event
*event
, int flags
)
222 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
223 struct hw_perf_event
*hwc
= &event
->hw
;
226 * ARM pmu always has to update the counter, so ignore
227 * PERF_EF_UPDATE, see comments in armpmu_start().
229 if (!(hwc
->state
& PERF_HES_STOPPED
)) {
230 armpmu
->disable(hwc
, hwc
->idx
);
231 barrier(); /* why? */
232 armpmu_event_update(event
, hwc
, hwc
->idx
);
233 hwc
->state
|= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
238 armpmu_start(struct perf_event
*event
, int flags
)
240 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
241 struct hw_perf_event
*hwc
= &event
->hw
;
244 * ARM pmu always has to reprogram the period, so ignore
245 * PERF_EF_RELOAD, see the comment below.
247 if (flags
& PERF_EF_RELOAD
)
248 WARN_ON_ONCE(!(hwc
->state
& PERF_HES_UPTODATE
));
252 * Set the period again. Some counters can't be stopped, so when we
253 * were stopped we simply disabled the IRQ source and the counter
254 * may have been left counting. If we don't do this step then we may
255 * get an interrupt too soon or *way* too late if the overflow has
256 * happened since disabling.
258 armpmu_event_set_period(event
, hwc
, hwc
->idx
);
259 armpmu
->enable(hwc
, hwc
->idx
);
263 armpmu_del(struct perf_event
*event
, int flags
)
265 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
266 struct pmu_hw_events
*hw_events
= armpmu
->get_hw_events();
267 struct hw_perf_event
*hwc
= &event
->hw
;
272 armpmu_stop(event
, PERF_EF_UPDATE
);
273 hw_events
->events
[idx
] = NULL
;
274 clear_bit(idx
, hw_events
->used_mask
);
276 perf_event_update_userpage(event
);
280 armpmu_add(struct perf_event
*event
, int flags
)
282 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
283 struct pmu_hw_events
*hw_events
= armpmu
->get_hw_events();
284 struct hw_perf_event
*hwc
= &event
->hw
;
288 perf_pmu_disable(event
->pmu
);
290 /* If we don't have a space for the counter then finish early. */
291 idx
= armpmu
->get_event_idx(hw_events
, hwc
);
298 * If there is an event in the counter we are going to use then make
299 * sure it is disabled.
302 armpmu
->disable(hwc
, idx
);
303 hw_events
->events
[idx
] = event
;
305 hwc
->state
= PERF_HES_STOPPED
| PERF_HES_UPTODATE
;
306 if (flags
& PERF_EF_START
)
307 armpmu_start(event
, PERF_EF_RELOAD
);
309 /* Propagate our changes to the userspace mapping. */
310 perf_event_update_userpage(event
);
313 perf_pmu_enable(event
->pmu
);
318 validate_event(struct pmu
*pmu
, struct pmu_hw_events
*hw_events
,
319 struct perf_event
*event
)
321 struct arm_pmu
*armpmu
;
322 struct hw_perf_event fake_event
= event
->hw
;
323 struct pmu
*leader_pmu
= event
->group_leader
->pmu
;
325 if (is_software_event(event
))
329 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
330 * core perf code won't check that the pmu->ctx == leader->ctx
331 * until after pmu->event_init(event).
333 if (event
->pmu
!= pmu
)
336 if (event
->pmu
!= leader_pmu
|| event
->state
< PERF_EVENT_STATE_OFF
)
339 if (event
->state
== PERF_EVENT_STATE_OFF
&& !event
->attr
.enable_on_exec
)
342 armpmu
= to_arm_pmu(event
->pmu
);
343 return armpmu
->get_event_idx(hw_events
, &fake_event
) >= 0;
347 validate_group(struct perf_event
*event
)
349 struct perf_event
*sibling
, *leader
= event
->group_leader
;
350 struct pmu_hw_events fake_pmu
;
351 DECLARE_BITMAP(fake_used_mask
, ARMPMU_MAX_HWEVENTS
);
354 * Initialise the fake PMU. We only need to populate the
355 * used_mask for the purposes of validation.
357 memset(fake_used_mask
, 0, sizeof(fake_used_mask
));
358 fake_pmu
.used_mask
= fake_used_mask
;
360 if (!validate_event(event
->pmu
, &fake_pmu
, leader
))
363 list_for_each_entry(sibling
, &leader
->sibling_list
, group_entry
) {
364 if (!validate_event(event
->pmu
, &fake_pmu
, sibling
))
368 if (!validate_event(event
->pmu
, &fake_pmu
, event
))
375 armpmu_release_hardware(struct arm_pmu
*armpmu
)
378 struct platform_device
*pmu_device
= armpmu
->plat_device
;
380 irqs
= min(pmu_device
->num_resources
, num_possible_cpus());
382 for (i
= 0; i
< irqs
; ++i
) {
383 if (!cpumask_test_and_clear_cpu(i
, &armpmu
->active_irqs
))
385 irq
= platform_get_irq(pmu_device
, i
);
387 free_irq(irq
, armpmu
);
392 armpmu_reserve_hardware(struct arm_pmu
*armpmu
)
394 int i
, err
, irq
, irqs
;
395 struct platform_device
*pmu_device
= armpmu
->plat_device
;
398 pr_err("no PMU device registered\n");
402 irqs
= min(pmu_device
->num_resources
, num_possible_cpus());
404 pr_err("no irqs for PMUs defined\n");
408 for (i
= 0; i
< irqs
; ++i
) {
410 irq
= platform_get_irq(pmu_device
, i
);
415 * If we have a single PMU interrupt that we can't shift,
416 * assume that we're running on a uniprocessor machine and
417 * continue. Otherwise, continue without this interrupt.
419 if (irq_set_affinity(irq
, cpumask_of(i
)) && irqs
> 1) {
420 pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
425 err
= request_irq(irq
, armpmu
->handle_irq
,
429 pr_err("unable to request IRQ%d for ARM PMU counters\n",
431 armpmu_release_hardware(armpmu
);
435 cpumask_set_cpu(i
, &armpmu
->active_irqs
);
442 hw_perf_event_destroy(struct perf_event
*event
)
444 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
445 atomic_t
*active_events
= &armpmu
->active_events
;
446 struct mutex
*pmu_reserve_mutex
= &armpmu
->reserve_mutex
;
448 if (atomic_dec_and_mutex_lock(active_events
, pmu_reserve_mutex
)) {
449 armpmu_release_hardware(armpmu
);
450 mutex_unlock(pmu_reserve_mutex
);
455 event_requires_mode_exclusion(struct perf_event_attr
*attr
)
457 return attr
->exclude_idle
|| attr
->exclude_user
||
458 attr
->exclude_kernel
|| attr
->exclude_hv
;
462 __hw_perf_event_init(struct perf_event
*event
)
464 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
465 struct hw_perf_event
*hwc
= &event
->hw
;
468 mapping
= armpmu
->map_event(event
);
471 pr_debug("event %x:%llx not supported\n", event
->attr
.type
,
477 * We don't assign an index until we actually place the event onto
478 * hardware. Use -1 to signify that we haven't decided where to put it
479 * yet. For SMP systems, each core has it's own PMU so we can't do any
480 * clever allocation or constraints checking at this point.
483 hwc
->config_base
= 0;
488 * Check whether we need to exclude the counter from certain modes.
490 if ((!armpmu
->set_event_filter
||
491 armpmu
->set_event_filter(hwc
, &event
->attr
)) &&
492 event_requires_mode_exclusion(&event
->attr
)) {
493 pr_debug("ARM performance counters do not support mode exclusion\n");
498 * Store the event encoding into the config_base field.
500 hwc
->config_base
|= (unsigned long)mapping
;
502 if (!hwc
->sample_period
) {
504 * For non-sampling runs, limit the sample_period to half
505 * of the counter width. That way, the new counter value
506 * is far less likely to overtake the previous one unless
507 * you have some serious IRQ latency issues.
509 hwc
->sample_period
= armpmu
->max_period
>> 1;
510 hwc
->last_period
= hwc
->sample_period
;
511 local64_set(&hwc
->period_left
, hwc
->sample_period
);
515 if (event
->group_leader
!= event
) {
516 err
= validate_group(event
);
524 static int armpmu_event_init(struct perf_event
*event
)
526 struct arm_pmu
*armpmu
= to_arm_pmu(event
->pmu
);
528 atomic_t
*active_events
= &armpmu
->active_events
;
530 if (armpmu
->map_event(event
) == -ENOENT
)
533 event
->destroy
= hw_perf_event_destroy
;
535 if (!atomic_inc_not_zero(active_events
)) {
536 mutex_lock(&armpmu
->reserve_mutex
);
537 if (atomic_read(active_events
) == 0)
538 err
= armpmu_reserve_hardware(armpmu
);
541 atomic_inc(active_events
);
542 mutex_unlock(&armpmu
->reserve_mutex
);
548 err
= __hw_perf_event_init(event
);
550 hw_perf_event_destroy(event
);
555 static void armpmu_enable(struct pmu
*pmu
)
557 struct arm_pmu
*armpmu
= to_arm_pmu(pmu
);
558 struct pmu_hw_events
*hw_events
= armpmu
->get_hw_events();
559 int enabled
= bitmap_weight(hw_events
->used_mask
, armpmu
->num_events
);
565 static void armpmu_disable(struct pmu
*pmu
)
567 struct arm_pmu
*armpmu
= to_arm_pmu(pmu
);
571 static void __init
armpmu_init(struct arm_pmu
*armpmu
)
573 atomic_set(&armpmu
->active_events
, 0);
574 mutex_init(&armpmu
->reserve_mutex
);
576 armpmu
->pmu
= (struct pmu
) {
577 .pmu_enable
= armpmu_enable
,
578 .pmu_disable
= armpmu_disable
,
579 .event_init
= armpmu_event_init
,
582 .start
= armpmu_start
,
588 int __init
armpmu_register(struct arm_pmu
*armpmu
, char *name
, int type
)
591 return perf_pmu_register(&armpmu
->pmu
, name
, type
);
595 * ARMv8 PMUv3 Performance Events handling code.
596 * Common event types.
598 enum armv8_pmuv3_perf_types
{
599 /* Required events. */
600 ARMV8_PMUV3_PERFCTR_PMNC_SW_INCR
= 0x00,
601 ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL
= 0x03,
602 ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS
= 0x04,
603 ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED
= 0x10,
604 ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES
= 0x11,
605 ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED
= 0x12,
607 /* At least one of the following is required. */
608 ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED
= 0x08,
609 ARMV8_PMUV3_PERFCTR_OP_SPEC
= 0x1B,
611 /* Common architectural events. */
612 ARMV8_PMUV3_PERFCTR_MEM_READ
= 0x06,
613 ARMV8_PMUV3_PERFCTR_MEM_WRITE
= 0x07,
614 ARMV8_PMUV3_PERFCTR_EXC_TAKEN
= 0x09,
615 ARMV8_PMUV3_PERFCTR_EXC_EXECUTED
= 0x0A,
616 ARMV8_PMUV3_PERFCTR_CID_WRITE
= 0x0B,
617 ARMV8_PMUV3_PERFCTR_PC_WRITE
= 0x0C,
618 ARMV8_PMUV3_PERFCTR_PC_IMM_BRANCH
= 0x0D,
619 ARMV8_PMUV3_PERFCTR_PC_PROC_RETURN
= 0x0E,
620 ARMV8_PMUV3_PERFCTR_MEM_UNALIGNED_ACCESS
= 0x0F,
621 ARMV8_PMUV3_PERFCTR_TTBR_WRITE
= 0x1C,
623 /* Common microarchitectural events. */
624 ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL
= 0x01,
625 ARMV8_PMUV3_PERFCTR_ITLB_REFILL
= 0x02,
626 ARMV8_PMUV3_PERFCTR_DTLB_REFILL
= 0x05,
627 ARMV8_PMUV3_PERFCTR_MEM_ACCESS
= 0x13,
628 ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS
= 0x14,
629 ARMV8_PMUV3_PERFCTR_L1_DCACHE_WB
= 0x15,
630 ARMV8_PMUV3_PERFCTR_L2_CACHE_ACCESS
= 0x16,
631 ARMV8_PMUV3_PERFCTR_L2_CACHE_REFILL
= 0x17,
632 ARMV8_PMUV3_PERFCTR_L2_CACHE_WB
= 0x18,
633 ARMV8_PMUV3_PERFCTR_BUS_ACCESS
= 0x19,
634 ARMV8_PMUV3_PERFCTR_MEM_ERROR
= 0x1A,
635 ARMV8_PMUV3_PERFCTR_BUS_CYCLES
= 0x1D,
638 /* PMUv3 HW events mapping. */
639 static const unsigned armv8_pmuv3_perf_map
[PERF_COUNT_HW_MAX
] = {
640 [PERF_COUNT_HW_CPU_CYCLES
] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES
,
641 [PERF_COUNT_HW_INSTRUCTIONS
] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED
,
642 [PERF_COUNT_HW_CACHE_REFERENCES
] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS
,
643 [PERF_COUNT_HW_CACHE_MISSES
] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL
,
644 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS
] = HW_OP_UNSUPPORTED
,
645 [PERF_COUNT_HW_BRANCH_MISSES
] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED
,
646 [PERF_COUNT_HW_BUS_CYCLES
] = HW_OP_UNSUPPORTED
,
647 [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND
] = HW_OP_UNSUPPORTED
,
648 [PERF_COUNT_HW_STALLED_CYCLES_BACKEND
] = HW_OP_UNSUPPORTED
,
651 static const unsigned armv8_pmuv3_perf_cache_map
[PERF_COUNT_HW_CACHE_MAX
]
652 [PERF_COUNT_HW_CACHE_OP_MAX
]
653 [PERF_COUNT_HW_CACHE_RESULT_MAX
] = {
656 [C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS
,
657 [C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL
,
660 [C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS
,
661 [C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL
,
664 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
665 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
670 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
671 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
674 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
675 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
678 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
679 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
684 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
685 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
688 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
689 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
692 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
693 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
698 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
699 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
702 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
703 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
706 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
707 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
712 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
713 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
716 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
717 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
720 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
721 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
726 [C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED
,
727 [C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED
,
730 [C(RESULT_ACCESS
)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED
,
731 [C(RESULT_MISS
)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED
,
734 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
735 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
740 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
741 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
744 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
745 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
748 [C(RESULT_ACCESS
)] = CACHE_OP_UNSUPPORTED
,
749 [C(RESULT_MISS
)] = CACHE_OP_UNSUPPORTED
,
755 * Perf Events' indices
757 #define ARMV8_IDX_CYCLE_COUNTER 0
758 #define ARMV8_IDX_COUNTER0 1
759 #define ARMV8_IDX_COUNTER_LAST (ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
761 #define ARMV8_MAX_COUNTERS 32
762 #define ARMV8_COUNTER_MASK (ARMV8_MAX_COUNTERS - 1)
765 * ARMv8 low level PMU access
769 * Perf Event to low level counters mapping
771 #define ARMV8_IDX_TO_COUNTER(x) \
772 (((x) - ARMV8_IDX_COUNTER0) & ARMV8_COUNTER_MASK)
775 * Per-CPU PMCR: config reg
777 #define ARMV8_PMCR_E (1 << 0) /* Enable all counters */
778 #define ARMV8_PMCR_P (1 << 1) /* Reset all counters */
779 #define ARMV8_PMCR_C (1 << 2) /* Cycle counter reset */
780 #define ARMV8_PMCR_D (1 << 3) /* CCNT counts every 64th cpu cycle */
781 #define ARMV8_PMCR_X (1 << 4) /* Export to ETM */
782 #define ARMV8_PMCR_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
783 #define ARMV8_PMCR_N_SHIFT 11 /* Number of counters supported */
784 #define ARMV8_PMCR_N_MASK 0x1f
785 #define ARMV8_PMCR_MASK 0x3f /* Mask for writable bits */
788 * PMOVSR: counters overflow flag status reg
790 #define ARMV8_OVSR_MASK 0xffffffff /* Mask for writable bits */
791 #define ARMV8_OVERFLOWED_MASK ARMV8_OVSR_MASK
794 * PMXEVTYPER: Event selection reg
796 #define ARMV8_EVTYPE_MASK 0xc80000ff /* Mask for writable bits */
797 #define ARMV8_EVTYPE_EVENT 0xff /* Mask for EVENT bits */
800 * Event filters for PMUv3
802 #define ARMV8_EXCLUDE_EL1 (1 << 31)
803 #define ARMV8_EXCLUDE_EL0 (1 << 30)
804 #define ARMV8_INCLUDE_EL2 (1 << 27)
806 static inline u32
armv8pmu_pmcr_read(void)
809 asm volatile("mrs %0, pmcr_el0" : "=r" (val
));
813 static inline void armv8pmu_pmcr_write(u32 val
)
815 val
&= ARMV8_PMCR_MASK
;
817 asm volatile("msr pmcr_el0, %0" :: "r" (val
));
820 static inline int armv8pmu_has_overflowed(u32 pmovsr
)
822 return pmovsr
& ARMV8_OVERFLOWED_MASK
;
825 static inline int armv8pmu_counter_valid(int idx
)
827 return idx
>= ARMV8_IDX_CYCLE_COUNTER
&& idx
<= ARMV8_IDX_COUNTER_LAST
;
830 static inline int armv8pmu_counter_has_overflowed(u32 pmnc
, int idx
)
835 if (!armv8pmu_counter_valid(idx
)) {
836 pr_err("CPU%u checking wrong counter %d overflow status\n",
837 smp_processor_id(), idx
);
839 counter
= ARMV8_IDX_TO_COUNTER(idx
);
840 ret
= pmnc
& BIT(counter
);
846 static inline int armv8pmu_select_counter(int idx
)
850 if (!armv8pmu_counter_valid(idx
)) {
851 pr_err("CPU%u selecting wrong PMNC counter %d\n",
852 smp_processor_id(), idx
);
856 counter
= ARMV8_IDX_TO_COUNTER(idx
);
857 asm volatile("msr pmselr_el0, %0" :: "r" (counter
));
863 static inline u32
armv8pmu_read_counter(int idx
)
867 if (!armv8pmu_counter_valid(idx
))
868 pr_err("CPU%u reading wrong counter %d\n",
869 smp_processor_id(), idx
);
870 else if (idx
== ARMV8_IDX_CYCLE_COUNTER
)
871 asm volatile("mrs %0, pmccntr_el0" : "=r" (value
));
872 else if (armv8pmu_select_counter(idx
) == idx
)
873 asm volatile("mrs %0, pmxevcntr_el0" : "=r" (value
));
878 static inline void armv8pmu_write_counter(int idx
, u32 value
)
880 if (!armv8pmu_counter_valid(idx
))
881 pr_err("CPU%u writing wrong counter %d\n",
882 smp_processor_id(), idx
);
883 else if (idx
== ARMV8_IDX_CYCLE_COUNTER
)
884 asm volatile("msr pmccntr_el0, %0" :: "r" (value
));
885 else if (armv8pmu_select_counter(idx
) == idx
)
886 asm volatile("msr pmxevcntr_el0, %0" :: "r" (value
));
889 static inline void armv8pmu_write_evtype(int idx
, u32 val
)
891 if (armv8pmu_select_counter(idx
) == idx
) {
892 val
&= ARMV8_EVTYPE_MASK
;
893 asm volatile("msr pmxevtyper_el0, %0" :: "r" (val
));
897 static inline int armv8pmu_enable_counter(int idx
)
901 if (!armv8pmu_counter_valid(idx
)) {
902 pr_err("CPU%u enabling wrong PMNC counter %d\n",
903 smp_processor_id(), idx
);
907 counter
= ARMV8_IDX_TO_COUNTER(idx
);
908 asm volatile("msr pmcntenset_el0, %0" :: "r" (BIT(counter
)));
912 static inline int armv8pmu_disable_counter(int idx
)
916 if (!armv8pmu_counter_valid(idx
)) {
917 pr_err("CPU%u disabling wrong PMNC counter %d\n",
918 smp_processor_id(), idx
);
922 counter
= ARMV8_IDX_TO_COUNTER(idx
);
923 asm volatile("msr pmcntenclr_el0, %0" :: "r" (BIT(counter
)));
927 static inline int armv8pmu_enable_intens(int idx
)
931 if (!armv8pmu_counter_valid(idx
)) {
932 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
933 smp_processor_id(), idx
);
937 counter
= ARMV8_IDX_TO_COUNTER(idx
);
938 asm volatile("msr pmintenset_el1, %0" :: "r" (BIT(counter
)));
942 static inline int armv8pmu_disable_intens(int idx
)
946 if (!armv8pmu_counter_valid(idx
)) {
947 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
948 smp_processor_id(), idx
);
952 counter
= ARMV8_IDX_TO_COUNTER(idx
);
953 asm volatile("msr pmintenclr_el1, %0" :: "r" (BIT(counter
)));
955 /* Clear the overflow flag in case an interrupt is pending. */
956 asm volatile("msr pmovsclr_el0, %0" :: "r" (BIT(counter
)));
961 static inline u32
armv8pmu_getreset_flags(void)
966 asm volatile("mrs %0, pmovsclr_el0" : "=r" (value
));
968 /* Write to clear flags */
969 value
&= ARMV8_OVSR_MASK
;
970 asm volatile("msr pmovsclr_el0, %0" :: "r" (value
));
975 static void armv8pmu_enable_event(struct hw_perf_event
*hwc
, int idx
)
978 struct pmu_hw_events
*events
= cpu_pmu
->get_hw_events();
981 * Enable counter and interrupt, and set the counter to count
982 * the event that we're interested in.
984 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
989 armv8pmu_disable_counter(idx
);
992 * Set event (if destined for PMNx counters).
994 armv8pmu_write_evtype(idx
, hwc
->config_base
);
997 * Enable interrupt for this counter
999 armv8pmu_enable_intens(idx
);
1004 armv8pmu_enable_counter(idx
);
1006 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
1009 static void armv8pmu_disable_event(struct hw_perf_event
*hwc
, int idx
)
1011 unsigned long flags
;
1012 struct pmu_hw_events
*events
= cpu_pmu
->get_hw_events();
1015 * Disable counter and interrupt
1017 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
1022 armv8pmu_disable_counter(idx
);
1025 * Disable interrupt for this counter
1027 armv8pmu_disable_intens(idx
);
1029 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
1032 static irqreturn_t
armv8pmu_handle_irq(int irq_num
, void *dev
)
1035 struct perf_sample_data data
;
1036 struct pmu_hw_events
*cpuc
;
1037 struct pt_regs
*regs
;
1041 * Get and reset the IRQ flags
1043 pmovsr
= armv8pmu_getreset_flags();
1046 * Did an overflow occur?
1048 if (!armv8pmu_has_overflowed(pmovsr
))
1052 * Handle the counter(s) overflow(s)
1054 regs
= get_irq_regs();
1056 cpuc
= &__get_cpu_var(cpu_hw_events
);
1057 for (idx
= 0; idx
< cpu_pmu
->num_events
; ++idx
) {
1058 struct perf_event
*event
= cpuc
->events
[idx
];
1059 struct hw_perf_event
*hwc
;
1061 /* Ignore if we don't have an event. */
1066 * We have a single interrupt for all counters. Check that
1067 * each counter has overflowed before we process it.
1069 if (!armv8pmu_counter_has_overflowed(pmovsr
, idx
))
1073 armpmu_event_update(event
, hwc
, idx
);
1074 perf_sample_data_init(&data
, 0, hwc
->last_period
);
1075 if (!armpmu_event_set_period(event
, hwc
, idx
))
1078 if (perf_event_overflow(event
, &data
, regs
))
1079 cpu_pmu
->disable(hwc
, idx
);
1083 * Handle the pending perf events.
1085 * Note: this call *must* be run with interrupts disabled. For
1086 * platforms that can have the PMU interrupts raised as an NMI, this
1094 static void armv8pmu_start(void)
1096 unsigned long flags
;
1097 struct pmu_hw_events
*events
= cpu_pmu
->get_hw_events();
1099 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
1100 /* Enable all counters */
1101 armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMCR_E
);
1102 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
1105 static void armv8pmu_stop(void)
1107 unsigned long flags
;
1108 struct pmu_hw_events
*events
= cpu_pmu
->get_hw_events();
1110 raw_spin_lock_irqsave(&events
->pmu_lock
, flags
);
1111 /* Disable all counters */
1112 armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMCR_E
);
1113 raw_spin_unlock_irqrestore(&events
->pmu_lock
, flags
);
1116 static int armv8pmu_get_event_idx(struct pmu_hw_events
*cpuc
,
1117 struct hw_perf_event
*event
)
1120 unsigned long evtype
= event
->config_base
& ARMV8_EVTYPE_EVENT
;
1122 /* Always place a cycle counter into the cycle counter. */
1123 if (evtype
== ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES
) {
1124 if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER
, cpuc
->used_mask
))
1127 return ARMV8_IDX_CYCLE_COUNTER
;
1131 * For anything other than a cycle counter, try and use
1132 * the events counters
1134 for (idx
= ARMV8_IDX_COUNTER0
; idx
< cpu_pmu
->num_events
; ++idx
) {
1135 if (!test_and_set_bit(idx
, cpuc
->used_mask
))
1139 /* The counters are all in use. */
1144 * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1146 static int armv8pmu_set_event_filter(struct hw_perf_event
*event
,
1147 struct perf_event_attr
*attr
)
1149 unsigned long config_base
= 0;
1151 if (attr
->exclude_idle
)
1153 if (attr
->exclude_user
)
1154 config_base
|= ARMV8_EXCLUDE_EL0
;
1155 if (attr
->exclude_kernel
)
1156 config_base
|= ARMV8_EXCLUDE_EL1
;
1157 if (!attr
->exclude_hv
)
1158 config_base
|= ARMV8_INCLUDE_EL2
;
1161 * Install the filter into config_base as this is used to
1162 * construct the event type.
1164 event
->config_base
= config_base
;
1169 static void armv8pmu_reset(void *info
)
1171 u32 idx
, nb_cnt
= cpu_pmu
->num_events
;
1173 /* The counter and interrupt enable registers are unknown at reset. */
1174 for (idx
= ARMV8_IDX_CYCLE_COUNTER
; idx
< nb_cnt
; ++idx
)
1175 armv8pmu_disable_event(NULL
, idx
);
1177 /* Initialize & Reset PMNC: C and P bits. */
1178 armv8pmu_pmcr_write(ARMV8_PMCR_P
| ARMV8_PMCR_C
);
1180 /* Disable access from userspace. */
1181 asm volatile("msr pmuserenr_el0, %0" :: "r" (0));
1184 static int armv8_pmuv3_map_event(struct perf_event
*event
)
1186 return map_cpu_event(event
, &armv8_pmuv3_perf_map
,
1187 &armv8_pmuv3_perf_cache_map
, 0xFF);
1190 static struct arm_pmu armv8pmu
= {
1191 .handle_irq
= armv8pmu_handle_irq
,
1192 .enable
= armv8pmu_enable_event
,
1193 .disable
= armv8pmu_disable_event
,
1194 .read_counter
= armv8pmu_read_counter
,
1195 .write_counter
= armv8pmu_write_counter
,
1196 .get_event_idx
= armv8pmu_get_event_idx
,
1197 .start
= armv8pmu_start
,
1198 .stop
= armv8pmu_stop
,
1199 .reset
= armv8pmu_reset
,
1200 .max_period
= (1LLU << 32) - 1,
1203 static u32 __init
armv8pmu_read_num_pmnc_events(void)
1207 /* Read the nb of CNTx counters supported from PMNC */
1208 nb_cnt
= (armv8pmu_pmcr_read() >> ARMV8_PMCR_N_SHIFT
) & ARMV8_PMCR_N_MASK
;
1210 /* Add the CPU cycles counter and return */
1214 static struct arm_pmu
*__init
armv8_pmuv3_pmu_init(void)
1216 armv8pmu
.name
= "arm/armv8-pmuv3";
1217 armv8pmu
.map_event
= armv8_pmuv3_map_event
;
1218 armv8pmu
.num_events
= armv8pmu_read_num_pmnc_events();
1219 armv8pmu
.set_event_filter
= armv8pmu_set_event_filter
;
1224 * Ensure the PMU has sane values out of reset.
1225 * This requires SMP to be available, so exists as a separate initcall.
1230 if (cpu_pmu
&& cpu_pmu
->reset
)
1231 return on_each_cpu(cpu_pmu
->reset
, NULL
, 1);
1234 arch_initcall(cpu_pmu_reset
);
1237 * PMU platform driver and devicetree bindings.
1239 static struct of_device_id armpmu_of_device_ids
[] = {
1240 {.compatible
= "arm,armv8-pmuv3"},
1244 static int armpmu_device_probe(struct platform_device
*pdev
)
1249 cpu_pmu
->plat_device
= pdev
;
1253 static struct platform_driver armpmu_driver
= {
1256 .of_match_table
= armpmu_of_device_ids
,
1258 .probe
= armpmu_device_probe
,
1261 static int __init
register_pmu_driver(void)
1263 return platform_driver_register(&armpmu_driver
);
1265 device_initcall(register_pmu_driver
);
1267 static struct pmu_hw_events
*armpmu_get_cpu_events(void)
1269 return &__get_cpu_var(cpu_hw_events
);
1272 static void __init
cpu_pmu_init(struct arm_pmu
*armpmu
)
1275 for_each_possible_cpu(cpu
) {
1276 struct pmu_hw_events
*events
= &per_cpu(cpu_hw_events
, cpu
);
1277 events
->events
= per_cpu(hw_events
, cpu
);
1278 events
->used_mask
= per_cpu(used_mask
, cpu
);
1279 raw_spin_lock_init(&events
->pmu_lock
);
1281 armpmu
->get_hw_events
= armpmu_get_cpu_events
;
1284 static int __init
init_hw_perf_events(void)
1286 u64 dfr
= read_cpuid(ID_AA64DFR0_EL1
);
1288 switch ((dfr
>> 8) & 0xf) {
1289 case 0x1: /* PMUv3 */
1290 cpu_pmu
= armv8_pmuv3_pmu_init();
1295 pr_info("enabled with %s PMU driver, %d counters available\n",
1296 cpu_pmu
->name
, cpu_pmu
->num_events
);
1297 cpu_pmu_init(cpu_pmu
);
1298 armpmu_register(cpu_pmu
, "cpu", PERF_TYPE_RAW
);
1300 pr_info("no hardware support available\n");
1305 early_initcall(init_hw_perf_events
);
1308 * Callchain handling code.
1311 struct frame_tail __user
*fp
;
1313 } __attribute__((packed
));
1316 * Get the return address for a single stackframe and return a pointer to the
1319 static struct frame_tail __user
*
1320 user_backtrace(struct frame_tail __user
*tail
,
1321 struct perf_callchain_entry
*entry
)
1323 struct frame_tail buftail
;
1326 /* Also check accessibility of one struct frame_tail beyond */
1327 if (!access_ok(VERIFY_READ
, tail
, sizeof(buftail
)))
1330 pagefault_disable();
1331 err
= __copy_from_user_inatomic(&buftail
, tail
, sizeof(buftail
));
1337 perf_callchain_store(entry
, buftail
.lr
);
1340 * Frame pointers should strictly progress back up the stack
1341 * (towards higher addresses).
1343 if (tail
>= buftail
.fp
)
1349 void perf_callchain_user(struct perf_callchain_entry
*entry
,
1350 struct pt_regs
*regs
)
1352 struct frame_tail __user
*tail
;
1354 if (perf_guest_cbs
&& perf_guest_cbs
->is_in_guest()) {
1355 /* We don't support guest os callchain now */
1359 perf_callchain_store(entry
, regs
->pc
);
1360 tail
= (struct frame_tail __user
*)regs
->regs
[29];
1362 while (entry
->nr
< PERF_MAX_STACK_DEPTH
&&
1363 tail
&& !((unsigned long)tail
& 0xf))
1364 tail
= user_backtrace(tail
, entry
);
1368 * Gets called by walk_stackframe() for every stackframe. This will be called
1369 * whist unwinding the stackframe and is like a subroutine return so we use
1372 static int callchain_trace(struct stackframe
*frame
, void *data
)
1374 struct perf_callchain_entry
*entry
= data
;
1375 perf_callchain_store(entry
, frame
->pc
);
1379 void perf_callchain_kernel(struct perf_callchain_entry
*entry
,
1380 struct pt_regs
*regs
)
1382 struct stackframe frame
;
1384 if (perf_guest_cbs
&& perf_guest_cbs
->is_in_guest()) {
1385 /* We don't support guest os callchain now */
1389 frame
.fp
= regs
->regs
[29];
1390 frame
.sp
= regs
->sp
;
1391 frame
.pc
= regs
->pc
;
1392 walk_stackframe(&frame
, callchain_trace
, entry
);
1395 unsigned long perf_instruction_pointer(struct pt_regs
*regs
)
1397 if (perf_guest_cbs
&& perf_guest_cbs
->is_in_guest())
1398 return perf_guest_cbs
->get_guest_ip();
1400 return instruction_pointer(regs
);
1403 unsigned long perf_misc_flags(struct pt_regs
*regs
)
1407 if (perf_guest_cbs
&& perf_guest_cbs
->is_in_guest()) {
1408 if (perf_guest_cbs
->is_user_mode())
1409 misc
|= PERF_RECORD_MISC_GUEST_USER
;
1411 misc
|= PERF_RECORD_MISC_GUEST_KERNEL
;
1413 if (user_mode(regs
))
1414 misc
|= PERF_RECORD_MISC_USER
;
1416 misc
|= PERF_RECORD_MISC_KERNEL
;