arch/arm/kernel/perf_event_xscale.c

   1 /*
   2  * ARMv5 [xscale] Performance counter handling code.
   3  *
   4  * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com>
   5  *
   6  * Based on the previous xscale OProfile code.
   7  *
   8  * There are two variants of the xscale PMU that we support:
   9  *      - xscale1pmu: 2 event counters and a cycle counter
  10  *      - xscale2pmu: 4 event counters and a cycle counter
  11  * The two variants share event definitions, but have different
  12  * PMU structures.
  13  */
  14
  15 #ifdef CONFIG_CPU_XSCALE
  16 enum xscale_perf_types {
  17         XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
  18         XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
  19         XSCALE_PERFCTR_DATA_STALL               = 0x02,
  20         XSCALE_PERFCTR_ITLB_MISS                = 0x03,
  21         XSCALE_PERFCTR_DTLB_MISS                = 0x04,
  22         XSCALE_PERFCTR_BRANCH                   = 0x05,
  23         XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
  24         XSCALE_PERFCTR_INSTRUCTION              = 0x07,
  25         XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
  26         XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
  27         XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
  28         XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
  29         XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
  30         XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
  31         XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
  32         XSCALE_PERFCTR_BCU_FULL                 = 0x11,
  33         XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
  34         XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
  35         XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
  36         XSCALE_PERFCTR_RMW                      = 0x16,
  37         /* XSCALE_PERFCTR_CCNT is not hardware defined */
  38         XSCALE_PERFCTR_CCNT                     = 0xFE,
  39         XSCALE_PERFCTR_UNUSED                   = 0xFF,
  40 };
  41
  42 enum xscale_counters {
  43         XSCALE_CYCLE_COUNTER    = 1,
  44         XSCALE_COUNTER0,
  45         XSCALE_COUNTER1,
  46         XSCALE_COUNTER2,
  47         XSCALE_COUNTER3,
  48 };
  49
  50 static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
  51         [PERF_COUNT_HW_CPU_CYCLES]          = XSCALE_PERFCTR_CCNT,
  52         [PERF_COUNT_HW_INSTRUCTIONS]        = XSCALE_PERFCTR_INSTRUCTION,
  53         [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
  54         [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
  55         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
  56         [PERF_COUNT_HW_BRANCH_MISSES]       = XSCALE_PERFCTR_BRANCH_MISS,
  57         [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
  58 };
  59
  60 static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
  61                                            [PERF_COUNT_HW_CACHE_OP_MAX]
  62                                            [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
  63         [C(L1D)] = {
  64                 [C(OP_READ)] = {
  65                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  66                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  67                 },
  68                 [C(OP_WRITE)] = {
  69                         [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
  70                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
  71                 },
  72                 [C(OP_PREFETCH)] = {
  73                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  74                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  75                 },
  76         },
  77         [C(L1I)] = {
  78                 [C(OP_READ)] = {
  79                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  80                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  81                 },
  82                 [C(OP_WRITE)] = {
  83                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  84                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
  85                 },
  86                 [C(OP_PREFETCH)] = {
  87                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  88                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  89                 },
  90         },
  91         [C(LL)] = {
  92                 [C(OP_READ)] = {
  93                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  94                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  95                 },
  96                 [C(OP_WRITE)] = {
  97                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  98                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  99                 },
 100                 [C(OP_PREFETCH)] = {
 101                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 102                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 103                 },
 104         },
 105         [C(DTLB)] = {
 106                 [C(OP_READ)] = {
 107                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 108                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 109                 },
 110                 [C(OP_WRITE)] = {
 111                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 112                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
 113                 },
 114                 [C(OP_PREFETCH)] = {
 115                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 116                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 117                 },
 118         },
 119         [C(ITLB)] = {
 120                 [C(OP_READ)] = {
 121                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 122                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 123                 },
 124                 [C(OP_WRITE)] = {
 125                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 126                         [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
 127                 },
 128                 [C(OP_PREFETCH)] = {
 129                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 130                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 131                 },
 132         },
 133         [C(BPU)] = {
 134                 [C(OP_READ)] = {
 135                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 136                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 137                 },
 138                 [C(OP_WRITE)] = {
 139                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 140                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 141                 },
 142                 [C(OP_PREFETCH)] = {
 143                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 144                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 145                 },
 146         },
 147         [C(NODE)] = {
 148                 [C(OP_READ)] = {
 149                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 150                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 151                 },
 152                 [C(OP_WRITE)] = {
 153                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 154                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 155                 },
 156                 [C(OP_PREFETCH)] = {
 157                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 158                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 159                 },
 160         },
 161 };
 162
 163 #define XSCALE_PMU_ENABLE       0x001
 164 #define XSCALE_PMN_RESET        0x002
 165 #define XSCALE_CCNT_RESET       0x004
 166 #define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
 167 #define XSCALE_PMU_CNT64        0x008
 168
 169 #define XSCALE1_OVERFLOWED_MASK 0x700
 170 #define XSCALE1_CCOUNT_OVERFLOW 0x400
 171 #define XSCALE1_COUNT0_OVERFLOW 0x100
 172 #define XSCALE1_COUNT1_OVERFLOW 0x200
 173 #define XSCALE1_CCOUNT_INT_EN   0x040
 174 #define XSCALE1_COUNT0_INT_EN   0x010
 175 #define XSCALE1_COUNT1_INT_EN   0x020
 176 #define XSCALE1_COUNT0_EVT_SHFT 12
 177 #define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
 178 #define XSCALE1_COUNT1_EVT_SHFT 20
 179 #define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
 180
 181 static inline u32
 182 xscale1pmu_read_pmnc(void)
 183 {
 184         u32 val;
 185         asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
 186         return val;
 187 }
 188
 189 static inline void
 190 xscale1pmu_write_pmnc(u32 val)
 191 {
 192         /* upper 4bits and 7, 11 are write-as-0 */
 193         val &= 0xffff77f;
 194         asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
 195 }
 196
 197 static inline int
 198 xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
 199                                         enum xscale_counters counter)
 200 {
 201         int ret = 0;
 202
 203         switch (counter) {
 204         case XSCALE_CYCLE_COUNTER:
 205                 ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
 206                 break;
 207         case XSCALE_COUNTER0:
 208                 ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
 209                 break;
 210         case XSCALE_COUNTER1:
 211                 ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
 212                 break;
 213         default:
 214                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 215         }
 216
 217         return ret;
 218 }
 219
 220 static irqreturn_t
 221 xscale1pmu_handle_irq(int irq_num, void *dev)
 222 {
 223         unsigned long pmnc;
 224         struct perf_sample_data data;
 225         struct cpu_hw_events *cpuc;
 226         struct pt_regs *regs;
 227         int idx;
 228
 229         /*
 230          * NOTE: there's an A stepping erratum that states if an overflow
 231          *       bit already exists and another occurs, the previous
 232          *       Overflow bit gets cleared. There's no workaround.
 233          *       Fixed in B stepping or later.
 234          */
 235         pmnc = xscale1pmu_read_pmnc();
 236
 237         /*
 238          * Write the value back to clear the overflow flags. Overflow
 239          * flags remain in pmnc for use below. We also disable the PMU
 240          * while we process the interrupt.
 241          */
 242         xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 243
 244         if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
 245                 return IRQ_NONE;
 246
 247         regs = get_irq_regs();
 248
 249         perf_sample_data_init(&data, 0);
 250
 251         cpuc = &__get_cpu_var(cpu_hw_events);
 252         for (idx = 0; idx <= armpmu->num_events; ++idx) {
 253                 struct perf_event *event = cpuc->events[idx];
 254                 struct hw_perf_event *hwc;
 255
 256                 if (!test_bit(idx, cpuc->active_mask))
 257                         continue;
 258
 259                 if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
 260                         continue;
 261
 262                 hwc = &event->hw;
 263                 armpmu_event_update(event, hwc, idx, 1);
 264                 data.period = event->hw.last_period;
 265                 if (!armpmu_event_set_period(event, hwc, idx))
 266                         continue;
 267
 268                 if (perf_event_overflow(event, &data, regs))
 269                         armpmu->disable(hwc, idx);
 270         }
 271
 272         irq_work_run();
 273
 274         /*
 275          * Re-enable the PMU.
 276          */
 277         pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 278         xscale1pmu_write_pmnc(pmnc);
 279
 280         return IRQ_HANDLED;
 281 }
 282
 283 static void
 284 xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
 285 {
 286         unsigned long val, mask, evt, flags;
 287
 288         switch (idx) {
 289         case XSCALE_CYCLE_COUNTER:
 290                 mask = 0;
 291                 evt = XSCALE1_CCOUNT_INT_EN;
 292                 break;
 293         case XSCALE_COUNTER0:
 294                 mask = XSCALE1_COUNT0_EVT_MASK;
 295                 evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
 296                         XSCALE1_COUNT0_INT_EN;
 297                 break;
 298         case XSCALE_COUNTER1:
 299                 mask = XSCALE1_COUNT1_EVT_MASK;
 300                 evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
 301                         XSCALE1_COUNT1_INT_EN;
 302                 break;
 303         default:
 304                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 305                 return;
 306         }
 307
 308         raw_spin_lock_irqsave(&pmu_lock, flags);
 309         val = xscale1pmu_read_pmnc();
 310         val &= ~mask;
 311         val |= evt;
 312         xscale1pmu_write_pmnc(val);
 313         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 314 }
 315
 316 static void
 317 xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
 318 {
 319         unsigned long val, mask, evt, flags;
 320
 321         switch (idx) {
 322         case XSCALE_CYCLE_COUNTER:
 323                 mask = XSCALE1_CCOUNT_INT_EN;
 324                 evt = 0;
 325                 break;
 326         case XSCALE_COUNTER0:
 327                 mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
 328                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
 329                 break;
 330         case XSCALE_COUNTER1:
 331                 mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
 332                 evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
 333                 break;
 334         default:
 335                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 336                 return;
 337         }
 338
 339         raw_spin_lock_irqsave(&pmu_lock, flags);
 340         val = xscale1pmu_read_pmnc();
 341         val &= ~mask;
 342         val |= evt;
 343         xscale1pmu_write_pmnc(val);
 344         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 345 }
 346
 347 static int
 348 xscale1pmu_get_event_idx(struct cpu_hw_events *cpuc,
 349                         struct hw_perf_event *event)
 350 {
 351         if (XSCALE_PERFCTR_CCNT == event->config_base) {
 352                 if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
 353                         return -EAGAIN;
 354
 355                 return XSCALE_CYCLE_COUNTER;
 356         } else {
 357                 if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask))
 358                         return XSCALE_COUNTER1;
 359
 360                 if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask))
 361                         return XSCALE_COUNTER0;
 362
 363                 return -EAGAIN;
 364         }
 365 }
 366
 367 static void
 368 xscale1pmu_start(void)
 369 {
 370         unsigned long flags, val;
 371
 372         raw_spin_lock_irqsave(&pmu_lock, flags);
 373         val = xscale1pmu_read_pmnc();
 374         val |= XSCALE_PMU_ENABLE;
 375         xscale1pmu_write_pmnc(val);
 376         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 377 }
 378
 379 static void
 380 xscale1pmu_stop(void)
 381 {
 382         unsigned long flags, val;
 383
 384         raw_spin_lock_irqsave(&pmu_lock, flags);
 385         val = xscale1pmu_read_pmnc();
 386         val &= ~XSCALE_PMU_ENABLE;
 387         xscale1pmu_write_pmnc(val);
 388         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 389 }
 390
 391 static inline u32
 392 xscale1pmu_read_counter(int counter)
 393 {
 394         u32 val = 0;
 395
 396         switch (counter) {
 397         case XSCALE_CYCLE_COUNTER:
 398                 asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
 399                 break;
 400         case XSCALE_COUNTER0:
 401                 asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
 402                 break;
 403         case XSCALE_COUNTER1:
 404                 asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
 405                 break;
 406         }
 407
 408         return val;
 409 }
 410
 411 static inline void
 412 xscale1pmu_write_counter(int counter, u32 val)
 413 {
 414         switch (counter) {
 415         case XSCALE_CYCLE_COUNTER:
 416                 asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
 417                 break;
 418         case XSCALE_COUNTER0:
 419                 asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
 420                 break;
 421         case XSCALE_COUNTER1:
 422                 asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
 423                 break;
 424         }
 425 }
 426
 427 static const struct arm_pmu xscale1pmu = {
 428         .id             = ARM_PERF_PMU_ID_XSCALE1,
 429         .name           = "xscale1",
 430         .handle_irq     = xscale1pmu_handle_irq,
 431         .enable         = xscale1pmu_enable_event,
 432         .disable        = xscale1pmu_disable_event,
 433         .read_counter   = xscale1pmu_read_counter,
 434         .write_counter  = xscale1pmu_write_counter,
 435         .get_event_idx  = xscale1pmu_get_event_idx,
 436         .start          = xscale1pmu_start,
 437         .stop           = xscale1pmu_stop,
 438         .cache_map      = &xscale_perf_cache_map,
 439         .event_map      = &xscale_perf_map,
 440         .raw_event_mask = 0xFF,
 441         .num_events     = 3,
 442         .max_period     = (1LLU << 32) - 1,
 443 };
 444
 445 static const struct arm_pmu *__init xscale1pmu_init(void)
 446 {
 447         return &xscale1pmu;
 448 }
 449
 450 #define XSCALE2_OVERFLOWED_MASK 0x01f
 451 #define XSCALE2_CCOUNT_OVERFLOW 0x001
 452 #define XSCALE2_COUNT0_OVERFLOW 0x002
 453 #define XSCALE2_COUNT1_OVERFLOW 0x004
 454 #define XSCALE2_COUNT2_OVERFLOW 0x008
 455 #define XSCALE2_COUNT3_OVERFLOW 0x010
 456 #define XSCALE2_CCOUNT_INT_EN   0x001
 457 #define XSCALE2_COUNT0_INT_EN   0x002
 458 #define XSCALE2_COUNT1_INT_EN   0x004
 459 #define XSCALE2_COUNT2_INT_EN   0x008
 460 #define XSCALE2_COUNT3_INT_EN   0x010
 461 #define XSCALE2_COUNT0_EVT_SHFT 0
 462 #define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
 463 #define XSCALE2_COUNT1_EVT_SHFT 8
 464 #define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
 465 #define XSCALE2_COUNT2_EVT_SHFT 16
 466 #define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
 467 #define XSCALE2_COUNT3_EVT_SHFT 24
 468 #define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
 469
 470 static inline u32
 471 xscale2pmu_read_pmnc(void)
 472 {
 473         u32 val;
 474         asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
 475         /* bits 1-2 and 4-23 are read-unpredictable */
 476         return val & 0xff000009;
 477 }
 478
 479 static inline void
 480 xscale2pmu_write_pmnc(u32 val)
 481 {
 482         /* bits 4-23 are write-as-0, 24-31 are write ignored */
 483         val &= 0xf;
 484         asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
 485 }
 486
 487 static inline u32
 488 xscale2pmu_read_overflow_flags(void)
 489 {
 490         u32 val;
 491         asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
 492         return val;
 493 }
 494
 495 static inline void
 496 xscale2pmu_write_overflow_flags(u32 val)
 497 {
 498         asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
 499 }
 500
 501 static inline u32
 502 xscale2pmu_read_event_select(void)
 503 {
 504         u32 val;
 505         asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
 506         return val;
 507 }
 508
 509 static inline void
 510 xscale2pmu_write_event_select(u32 val)
 511 {
 512         asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
 513 }
 514
 515 static inline u32
 516 xscale2pmu_read_int_enable(void)
 517 {
 518         u32 val;
 519         asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
 520         return val;
 521 }
 522
 523 static void
 524 xscale2pmu_write_int_enable(u32 val)
 525 {
 526         asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
 527 }
 528
 529 static inline int
 530 xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
 531                                         enum xscale_counters counter)
 532 {
 533         int ret = 0;
 534
 535         switch (counter) {
 536         case XSCALE_CYCLE_COUNTER:
 537                 ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
 538                 break;
 539         case XSCALE_COUNTER0:
 540                 ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
 541                 break;
 542         case XSCALE_COUNTER1:
 543                 ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
 544                 break;
 545         case XSCALE_COUNTER2:
 546                 ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
 547                 break;
 548         case XSCALE_COUNTER3:
 549                 ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
 550                 break;
 551         default:
 552                 WARN_ONCE(1, "invalid counter number (%d)\n", counter);
 553         }
 554
 555         return ret;
 556 }
 557
 558 static irqreturn_t
 559 xscale2pmu_handle_irq(int irq_num, void *dev)
 560 {
 561         unsigned long pmnc, of_flags;
 562         struct perf_sample_data data;
 563         struct cpu_hw_events *cpuc;
 564         struct pt_regs *regs;
 565         int idx;
 566
 567         /* Disable the PMU. */
 568         pmnc = xscale2pmu_read_pmnc();
 569         xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
 570
 571         /* Check the overflow flag register. */
 572         of_flags = xscale2pmu_read_overflow_flags();
 573         if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
 574                 return IRQ_NONE;
 575
 576         /* Clear the overflow bits. */
 577         xscale2pmu_write_overflow_flags(of_flags);
 578
 579         regs = get_irq_regs();
 580
 581         perf_sample_data_init(&data, 0);
 582
 583         cpuc = &__get_cpu_var(cpu_hw_events);
 584         for (idx = 0; idx <= armpmu->num_events; ++idx) {
 585                 struct perf_event *event = cpuc->events[idx];
 586                 struct hw_perf_event *hwc;
 587
 588                 if (!test_bit(idx, cpuc->active_mask))
 589                         continue;
 590
 591                 if (!xscale2_pmnc_counter_has_overflowed(pmnc, idx))
 592                         continue;
 593
 594                 hwc = &event->hw;
 595                 armpmu_event_update(event, hwc, idx, 1);
 596                 data.period = event->hw.last_period;
 597                 if (!armpmu_event_set_period(event, hwc, idx))
 598                         continue;
 599
 600                 if (perf_event_overflow(event, &data, regs))
 601                         armpmu->disable(hwc, idx);
 602         }
 603
 604         irq_work_run();
 605
 606         /*
 607          * Re-enable the PMU.
 608          */
 609         pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
 610         xscale2pmu_write_pmnc(pmnc);
 611
 612         return IRQ_HANDLED;
 613 }
 614
 615 static void
 616 xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
 617 {
 618         unsigned long flags, ien, evtsel;
 619
 620         ien = xscale2pmu_read_int_enable();
 621         evtsel = xscale2pmu_read_event_select();
 622
 623         switch (idx) {
 624         case XSCALE_CYCLE_COUNTER:
 625                 ien |= XSCALE2_CCOUNT_INT_EN;
 626                 break;
 627         case XSCALE_COUNTER0:
 628                 ien |= XSCALE2_COUNT0_INT_EN;
 629                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 630                 evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
 631                 break;
 632         case XSCALE_COUNTER1:
 633                 ien |= XSCALE2_COUNT1_INT_EN;
 634                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 635                 evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
 636                 break;
 637         case XSCALE_COUNTER2:
 638                 ien |= XSCALE2_COUNT2_INT_EN;
 639                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 640                 evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
 641                 break;
 642         case XSCALE_COUNTER3:
 643                 ien |= XSCALE2_COUNT3_INT_EN;
 644                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 645                 evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
 646                 break;
 647         default:
 648                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 649                 return;
 650         }
 651
 652         raw_spin_lock_irqsave(&pmu_lock, flags);
 653         xscale2pmu_write_event_select(evtsel);
 654         xscale2pmu_write_int_enable(ien);
 655         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 656 }
 657
 658 static void
 659 xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
 660 {
 661         unsigned long flags, ien, evtsel;
 662
 663         ien = xscale2pmu_read_int_enable();
 664         evtsel = xscale2pmu_read_event_select();
 665
 666         switch (idx) {
 667         case XSCALE_CYCLE_COUNTER:
 668                 ien &= ~XSCALE2_CCOUNT_INT_EN;
 669                 break;
 670         case XSCALE_COUNTER0:
 671                 ien &= ~XSCALE2_COUNT0_INT_EN;
 672                 evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
 673                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
 674                 break;
 675         case XSCALE_COUNTER1:
 676                 ien &= ~XSCALE2_COUNT1_INT_EN;
 677                 evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
 678                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
 679                 break;
 680         case XSCALE_COUNTER2:
 681                 ien &= ~XSCALE2_COUNT2_INT_EN;
 682                 evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
 683                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
 684                 break;
 685         case XSCALE_COUNTER3:
 686                 ien &= ~XSCALE2_COUNT3_INT_EN;
 687                 evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
 688                 evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
 689                 break;
 690         default:
 691                 WARN_ONCE(1, "invalid counter number (%d)\n", idx);
 692                 return;
 693         }
 694
 695         raw_spin_lock_irqsave(&pmu_lock, flags);
 696         xscale2pmu_write_event_select(evtsel);
 697         xscale2pmu_write_int_enable(ien);
 698         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 699 }
 700
 701 static int
 702 xscale2pmu_get_event_idx(struct cpu_hw_events *cpuc,
 703                         struct hw_perf_event *event)
 704 {
 705         int idx = xscale1pmu_get_event_idx(cpuc, event);
 706         if (idx >= 0)
 707                 goto out;
 708
 709         if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
 710                 idx = XSCALE_COUNTER3;
 711         else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
 712                 idx = XSCALE_COUNTER2;
 713 out:
 714         return idx;
 715 }
 716
 717 static void
 718 xscale2pmu_start(void)
 719 {
 720         unsigned long flags, val;
 721
 722         raw_spin_lock_irqsave(&pmu_lock, flags);
 723         val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
 724         val |= XSCALE_PMU_ENABLE;
 725         xscale2pmu_write_pmnc(val);
 726         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 727 }
 728
 729 static void
 730 xscale2pmu_stop(void)
 731 {
 732         unsigned long flags, val;
 733
 734         raw_spin_lock_irqsave(&pmu_lock, flags);
 735         val = xscale2pmu_read_pmnc();
 736         val &= ~XSCALE_PMU_ENABLE;
 737         xscale2pmu_write_pmnc(val);
 738         raw_spin_unlock_irqrestore(&pmu_lock, flags);
 739 }
 740
 741 static inline u32
 742 xscale2pmu_read_counter(int counter)
 743 {
 744         u32 val = 0;
 745
 746         switch (counter) {
 747         case XSCALE_CYCLE_COUNTER:
 748                 asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
 749                 break;
 750         case XSCALE_COUNTER0:
 751                 asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
 752                 break;
 753         case XSCALE_COUNTER1:
 754                 asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
 755                 break;
 756         case XSCALE_COUNTER2:
 757                 asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
 758                 break;
 759         case XSCALE_COUNTER3:
 760                 asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
 761                 break;
 762         }
 763
 764         return val;
 765 }
 766
 767 static inline void
 768 xscale2pmu_write_counter(int counter, u32 val)
 769 {
 770         switch (counter) {
 771         case XSCALE_CYCLE_COUNTER:
 772                 asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
 773                 break;
 774         case XSCALE_COUNTER0:
 775                 asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
 776                 break;
 777         case XSCALE_COUNTER1:
 778                 asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
 779                 break;
 780         case XSCALE_COUNTER2:
 781                 asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
 782                 break;
 783         case XSCALE_COUNTER3:
 784                 asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
 785                 break;
 786         }
 787 }
 788
 789 static const struct arm_pmu xscale2pmu = {
 790         .id             = ARM_PERF_PMU_ID_XSCALE2,
 791         .name           = "xscale2",
 792         .handle_irq     = xscale2pmu_handle_irq,
 793         .enable         = xscale2pmu_enable_event,
 794         .disable        = xscale2pmu_disable_event,
 795         .read_counter   = xscale2pmu_read_counter,
 796         .write_counter  = xscale2pmu_write_counter,
 797         .get_event_idx  = xscale2pmu_get_event_idx,
 798         .start          = xscale2pmu_start,
 799         .stop           = xscale2pmu_stop,
 800         .cache_map      = &xscale_perf_cache_map,
 801         .event_map      = &xscale_perf_map,
 802         .raw_event_mask = 0xFF,
 803         .num_events     = 5,
 804         .max_period     = (1LLU << 32) - 1,
 805 };
 806
 807 static const struct arm_pmu *__init xscale2pmu_init(void)
 808 {
 809         return &xscale2pmu;
 810 }
 811 #else
 812 static const struct arm_pmu *__init xscale1pmu_init(void)
 813 {
 814         return NULL;
 815 }
 816
 817 static const struct arm_pmu *__init xscale2pmu_init(void)
 818 {
 819         return NULL;
 820 }
 821 #endif  /* CONFIG_CPU_XSCALE */