arch/x86/kernel/cpu/perf_event_amd_ibs.c

   1 /*
   2  * Performance events - AMD IBS
   3  *
   4  *  Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter
   5  *
   6  *  For licencing details see kernel-base/COPYING
   7  */
   8
   9 #include <linux/perf_event.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12
  13 #include <asm/apic.h>
  14
  15 static u32 ibs_caps;
  16
  17 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
  18
  19 #include <linux/kprobes.h>
  20 #include <linux/hardirq.h>
  21
  22 #include <asm/nmi.h>
  23
  24 #define IBS_FETCH_CONFIG_MASK   (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT)
  25 #define IBS_OP_CONFIG_MASK      IBS_OP_MAX_CNT
  26
  27 struct perf_ibs {
  28         struct pmu      pmu;
  29         unsigned int    msr;
  30         u64             config_mask;
  31         u64             cnt_mask;
  32         u64             enable_mask;
  33         u64             valid_mask;
  34         unsigned long   offset_mask[1];
  35         int             offset_max;
  36 };
  37
  38 struct perf_ibs_data {
  39         u32             size;
  40         union {
  41                 u32     data[0];        /* data buffer starts here */
  42                 u32     caps;
  43         };
  44         u64             regs[MSR_AMD64_IBS_REG_COUNT_MAX];
  45 };
  46
  47 static struct perf_ibs perf_ibs_fetch;
  48 static struct perf_ibs perf_ibs_op;
  49
  50 static struct perf_ibs *get_ibs_pmu(int type)
  51 {
  52         if (perf_ibs_fetch.pmu.type == type)
  53                 return &perf_ibs_fetch;
  54         if (perf_ibs_op.pmu.type == type)
  55                 return &perf_ibs_op;
  56         return NULL;
  57 }
  58
  59 static int perf_ibs_init(struct perf_event *event)
  60 {
  61         struct hw_perf_event *hwc = &event->hw;
  62         struct perf_ibs *perf_ibs;
  63         u64 max_cnt, config;
  64
  65         perf_ibs = get_ibs_pmu(event->attr.type);
  66         if (!perf_ibs)
  67                 return -ENOENT;
  68
  69         config = event->attr.config;
  70         if (config & ~perf_ibs->config_mask)
  71                 return -EINVAL;
  72
  73         if (hwc->sample_period) {
  74                 if (config & perf_ibs->cnt_mask)
  75                         /* raw max_cnt may not be set */
  76                         return -EINVAL;
  77                 if (hwc->sample_period & 0x0f)
  78                         /* lower 4 bits can not be set in ibs max cnt */
  79                         return -EINVAL;
  80                 max_cnt = hwc->sample_period >> 4;
  81                 if (max_cnt & ~perf_ibs->cnt_mask)
  82                         /* out of range */
  83                         return -EINVAL;
  84                 config |= max_cnt;
  85         } else {
  86                 max_cnt = config & perf_ibs->cnt_mask;
  87                 event->attr.sample_period = max_cnt << 4;
  88                 hwc->sample_period = event->attr.sample_period;
  89         }
  90
  91         if (!max_cnt)
  92                 return -EINVAL;
  93
  94         hwc->config_base = perf_ibs->msr;
  95         hwc->config = config;
  96
  97         return 0;
  98 }
  99
 100 static int perf_ibs_add(struct perf_event *event, int flags)
 101 {
 102         return 0;
 103 }
 104
 105 static void perf_ibs_del(struct perf_event *event, int flags)
 106 {
 107 }
 108
 109 static struct perf_ibs perf_ibs_fetch = {
 110         .pmu = {
 111                 .task_ctx_nr    = perf_invalid_context,
 112
 113                 .event_init     = perf_ibs_init,
 114                 .add            = perf_ibs_add,
 115                 .del            = perf_ibs_del,
 116         },
 117         .msr                    = MSR_AMD64_IBSFETCHCTL,
 118         .config_mask            = IBS_FETCH_CONFIG_MASK,
 119         .cnt_mask               = IBS_FETCH_MAX_CNT,
 120         .enable_mask            = IBS_FETCH_ENABLE,
 121         .valid_mask             = IBS_FETCH_VAL,
 122         .offset_mask            = { MSR_AMD64_IBSFETCH_REG_MASK },
 123         .offset_max             = MSR_AMD64_IBSFETCH_REG_COUNT,
 124 };
 125
 126 static struct perf_ibs perf_ibs_op = {
 127         .pmu = {
 128                 .task_ctx_nr    = perf_invalid_context,
 129
 130                 .event_init     = perf_ibs_init,
 131                 .add            = perf_ibs_add,
 132                 .del            = perf_ibs_del,
 133         },
 134         .msr                    = MSR_AMD64_IBSOPCTL,
 135         .config_mask            = IBS_OP_CONFIG_MASK,
 136         .cnt_mask               = IBS_OP_MAX_CNT,
 137         .enable_mask            = IBS_OP_ENABLE,
 138         .valid_mask             = IBS_OP_VAL,
 139         .offset_mask            = { MSR_AMD64_IBSOP_REG_MASK },
 140         .offset_max             = MSR_AMD64_IBSOP_REG_COUNT,
 141 };
 142
 143 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 144 {
 145         struct perf_event *event = NULL;
 146         struct hw_perf_event *hwc = &event->hw;
 147         struct perf_sample_data data;
 148         struct perf_raw_record raw;
 149         struct pt_regs regs;
 150         struct perf_ibs_data ibs_data;
 151         int offset, size;
 152         unsigned int msr;
 153         u64 *buf;
 154
 155         msr = hwc->config_base;
 156         buf = ibs_data.regs;
 157         rdmsrl(msr, *buf);
 158         if (!(*buf++ & perf_ibs->valid_mask))
 159                 return 0;
 160
 161         perf_sample_data_init(&data, 0);
 162         if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 163                 ibs_data.caps = ibs_caps;
 164                 size = 1;
 165                 offset = 1;
 166                 do {
 167                     rdmsrl(msr + offset, *buf++);
 168                     size++;
 169                     offset = find_next_bit(perf_ibs->offset_mask,
 170                                            perf_ibs->offset_max,
 171                                            offset + 1);
 172                 } while (offset < perf_ibs->offset_max);
 173                 raw.size = sizeof(u32) + sizeof(u64) * size;
 174                 raw.data = ibs_data.data;
 175                 data.raw = &raw;
 176         }
 177
 178         regs = *iregs; /* XXX: update ip from ibs sample */
 179
 180         if (perf_event_overflow(event, &data, &regs))
 181                 ; /* stop */
 182         else
 183                 /* reenable */
 184                 wrmsrl(hwc->config_base, hwc->config | perf_ibs->enable_mask);
 185
 186         return 1;
 187 }
 188
 189 static int __kprobes
 190 perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
 191 {
 192         int handled = 0;
 193
 194         handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs);
 195         handled += perf_ibs_handle_irq(&perf_ibs_op, regs);
 196
 197         if (handled)
 198                 inc_irq_stat(apic_perf_irqs);
 199
 200         return handled;
 201 }
 202
 203 static __init int perf_event_ibs_init(void)
 204 {
 205         if (!ibs_caps)
 206                 return -ENODEV; /* ibs not supported by the cpu */
 207
 208         perf_pmu_register(&perf_ibs_fetch.pmu, "ibs_fetch", -1);
 209         perf_pmu_register(&perf_ibs_op.pmu, "ibs_op", -1);
 210         register_nmi_handler(NMI_LOCAL, &perf_ibs_nmi_handler, 0, "perf_ibs");
 211         printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps);
 212
 213         return 0;
 214 }
 215
 216 #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */
 217
 218 static __init int perf_event_ibs_init(void) { return 0; }
 219
 220 #endif
 221
 222 /* IBS - apic initialization, for perf and oprofile */
 223
 224 static __init u32 __get_ibs_caps(void)
 225 {
 226         u32 caps;
 227         unsigned int max_level;
 228
 229         if (!boot_cpu_has(X86_FEATURE_IBS))
 230                 return 0;
 231
 232         /* check IBS cpuid feature flags */
 233         max_level = cpuid_eax(0x80000000);
 234         if (max_level < IBS_CPUID_FEATURES)
 235                 return IBS_CAPS_DEFAULT;
 236
 237         caps = cpuid_eax(IBS_CPUID_FEATURES);
 238         if (!(caps & IBS_CAPS_AVAIL))
 239                 /* cpuid flags not valid */
 240                 return IBS_CAPS_DEFAULT;
 241
 242         return caps;
 243 }
 244
 245 u32 get_ibs_caps(void)
 246 {
 247         return ibs_caps;
 248 }
 249
 250 EXPORT_SYMBOL(get_ibs_caps);
 251
 252 static inline int get_eilvt(int offset)
 253 {
 254         return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
 255 }
 256
 257 static inline int put_eilvt(int offset)
 258 {
 259         return !setup_APIC_eilvt(offset, 0, 0, 1);
 260 }
 261
 262 /*
 263  * Check and reserve APIC extended interrupt LVT offset for IBS if available.
 264  */
 265 static inline int ibs_eilvt_valid(void)
 266 {
 267         int offset;
 268         u64 val;
 269         int valid = 0;
 270
 271         preempt_disable();
 272
 273         rdmsrl(MSR_AMD64_IBSCTL, val);
 274         offset = val & IBSCTL_LVT_OFFSET_MASK;
 275
 276         if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
 277                 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
 278                        smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
 279                 goto out;
 280         }
 281
 282         if (!get_eilvt(offset)) {
 283                 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
 284                        smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
 285                 goto out;
 286         }
 287
 288         valid = 1;
 289 out:
 290         preempt_enable();
 291
 292         return valid;
 293 }
 294
 295 static int setup_ibs_ctl(int ibs_eilvt_off)
 296 {
 297         struct pci_dev *cpu_cfg;
 298         int nodes;
 299         u32 value = 0;
 300
 301         nodes = 0;
 302         cpu_cfg = NULL;
 303         do {
 304                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 305                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 306                                          cpu_cfg);
 307                 if (!cpu_cfg)
 308                         break;
 309                 ++nodes;
 310                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 311                                        | IBSCTL_LVT_OFFSET_VALID);
 312                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 313                 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
 314                         pci_dev_put(cpu_cfg);
 315                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 316                                "IBSCTL = 0x%08x\n", value);
 317                         return -EINVAL;
 318                 }
 319         } while (1);
 320
 321         if (!nodes) {
 322                 printk(KERN_DEBUG "No CPU node configured for IBS\n");
 323                 return -ENODEV;
 324         }
 325
 326         return 0;
 327 }
 328
 329 /*
 330  * This runs only on the current cpu. We try to find an LVT offset and
 331  * setup the local APIC. For this we must disable preemption. On
 332  * success we initialize all nodes with this offset. This updates then
 333  * the offset in the IBS_CTL per-node msr. The per-core APIC setup of
 334  * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that
 335  * is using the new offset.
 336  */
 337 static int force_ibs_eilvt_setup(void)
 338 {
 339         int offset;
 340         int ret;
 341
 342         preempt_disable();
 343         /* find the next free available EILVT entry, skip offset 0 */
 344         for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) {
 345                 if (get_eilvt(offset))
 346                         break;
 347         }
 348         preempt_enable();
 349
 350         if (offset == APIC_EILVT_NR_MAX) {
 351                 printk(KERN_DEBUG "No EILVT entry available\n");
 352                 return -EBUSY;
 353         }
 354
 355         ret = setup_ibs_ctl(offset);
 356         if (ret)
 357                 goto out;
 358
 359         if (!ibs_eilvt_valid()) {
 360                 ret = -EFAULT;
 361                 goto out;
 362         }
 363
 364         pr_info("IBS: LVT offset %d assigned\n", offset);
 365
 366         return 0;
 367 out:
 368         preempt_disable();
 369         put_eilvt(offset);
 370         preempt_enable();
 371         return ret;
 372 }
 373
 374 static inline int get_ibs_lvt_offset(void)
 375 {
 376         u64 val;
 377
 378         rdmsrl(MSR_AMD64_IBSCTL, val);
 379         if (!(val & IBSCTL_LVT_OFFSET_VALID))
 380                 return -EINVAL;
 381
 382         return val & IBSCTL_LVT_OFFSET_MASK;
 383 }
 384
 385 static void setup_APIC_ibs(void *dummy)
 386 {
 387         int offset;
 388
 389         offset = get_ibs_lvt_offset();
 390         if (offset < 0)
 391                 goto failed;
 392
 393         if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
 394                 return;
 395 failed:
 396         pr_warn("perf: IBS APIC setup failed on cpu #%d\n",
 397                 smp_processor_id());
 398 }
 399
 400 static void clear_APIC_ibs(void *dummy)
 401 {
 402         int offset;
 403
 404         offset = get_ibs_lvt_offset();
 405         if (offset >= 0)
 406                 setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
 407 }
 408
 409 static int __cpuinit
 410 perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
 411 {
 412         switch (action & ~CPU_TASKS_FROZEN) {
 413         case CPU_STARTING:
 414                 setup_APIC_ibs(NULL);
 415                 break;
 416         case CPU_DYING:
 417                 clear_APIC_ibs(NULL);
 418                 break;
 419         default:
 420                 break;
 421         }
 422
 423         return NOTIFY_OK;
 424 }
 425
 426 static __init int amd_ibs_init(void)
 427 {
 428         u32 caps;
 429         int ret = -EINVAL;
 430
 431         caps = __get_ibs_caps();
 432         if (!caps)
 433                 return -ENODEV; /* ibs not supported by the cpu */
 434
 435         /*
 436          * Force LVT offset assignment for family 10h: The offsets are
 437          * not assigned by the BIOS for this family, so the OS is
 438          * responsible for doing it. If the OS assignment fails, fall
 439          * back to BIOS settings and try to setup this.
 440          */
 441         if (boot_cpu_data.x86 == 0x10)
 442                 force_ibs_eilvt_setup();
 443
 444         if (!ibs_eilvt_valid())
 445                 goto out;
 446
 447         get_online_cpus();
 448         ibs_caps = caps;
 449         /* make ibs_caps visible to other cpus: */
 450         smp_mb();
 451         perf_cpu_notifier(perf_ibs_cpu_notifier);
 452         smp_call_function(setup_APIC_ibs, NULL, 1);
 453         put_online_cpus();
 454
 455         ret = perf_event_ibs_init();
 456 out:
 457         if (ret)
 458                 pr_err("Failed to setup IBS, %d\n", ret);
 459         return ret;
 460 }
 461
 462 /* Since we need the pci subsystem to init ibs we can't do this earlier: */
 463 device_initcall(amd_ibs_init);