arch/x86/kernel/alternative.c

   1 #include <linux/module.h>
   2 #include <linux/sched.h>
   3 #include <linux/spinlock.h>
   4 #include <linux/list.h>
   5 #include <linux/kprobes.h>
   6 #include <linux/mm.h>
   7 #include <linux/vmalloc.h>
   8 #include <asm/alternative.h>
   9 #include <asm/sections.h>
  10 #include <asm/pgtable.h>
  11 #include <asm/mce.h>
  12 #include <asm/nmi.h>
  13 #include <asm/vsyscall.h>
  14
  15 #define MAX_PATCH_LEN (255-1)
  16
  17 #ifdef CONFIG_HOTPLUG_CPU
  18 static int smp_alt_once;
  19
  20 static int __init bootonly(char *str)
  21 {
  22         smp_alt_once = 1;
  23         return 1;
  24 }
  25 __setup("smp-alt-boot", bootonly);
  26 #else
  27 #define smp_alt_once 1
  28 #endif
  29
  30 static int debug_alternative;
  31
  32 static int __init debug_alt(char *str)
  33 {
  34         debug_alternative = 1;
  35         return 1;
  36 }
  37 __setup("debug-alternative", debug_alt);
  38
  39 static int noreplace_smp;
  40
  41 static int __init setup_noreplace_smp(char *str)
  42 {
  43         noreplace_smp = 1;
  44         return 1;
  45 }
  46 __setup("noreplace-smp", setup_noreplace_smp);
  47
  48 #ifdef CONFIG_PARAVIRT
  49 static int noreplace_paravirt = 0;
  50
  51 static int __init setup_noreplace_paravirt(char *str)
  52 {
  53         noreplace_paravirt = 1;
  54         return 1;
  55 }
  56 __setup("noreplace-paravirt", setup_noreplace_paravirt);
  57 #endif
  58
  59 #define DPRINTK(fmt, args...) if (debug_alternative) \
  60         printk(KERN_DEBUG fmt, args)
  61
  62 #ifdef GENERIC_NOP1
  63 /* Use inline assembly to define this because the nops are defined
  64    as inline assembly strings in the include files and we cannot
  65    get them easily into strings. */
  66 asm("\t.data\nintelnops: "
  67         GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
  68         GENERIC_NOP7 GENERIC_NOP8);
  69 extern unsigned char intelnops[];
  70 static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
  71         NULL,
  72         intelnops,
  73         intelnops + 1,
  74         intelnops + 1 + 2,
  75         intelnops + 1 + 2 + 3,
  76         intelnops + 1 + 2 + 3 + 4,
  77         intelnops + 1 + 2 + 3 + 4 + 5,
  78         intelnops + 1 + 2 + 3 + 4 + 5 + 6,
  79         intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  80 };
  81 #endif
  82
  83 #ifdef K8_NOP1
  84 asm("\t.data\nk8nops: "
  85         K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
  86         K8_NOP7 K8_NOP8);
  87 extern unsigned char k8nops[];
  88 static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
  89         NULL,
  90         k8nops,
  91         k8nops + 1,
  92         k8nops + 1 + 2,
  93         k8nops + 1 + 2 + 3,
  94         k8nops + 1 + 2 + 3 + 4,
  95         k8nops + 1 + 2 + 3 + 4 + 5,
  96         k8nops + 1 + 2 + 3 + 4 + 5 + 6,
  97         k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
  98 };
  99 #endif
 100
 101 #ifdef K7_NOP1
 102 asm("\t.data\nk7nops: "
 103         K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
 104         K7_NOP7 K7_NOP8);
 105 extern unsigned char k7nops[];
 106 static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
 107         NULL,
 108         k7nops,
 109         k7nops + 1,
 110         k7nops + 1 + 2,
 111         k7nops + 1 + 2 + 3,
 112         k7nops + 1 + 2 + 3 + 4,
 113         k7nops + 1 + 2 + 3 + 4 + 5,
 114         k7nops + 1 + 2 + 3 + 4 + 5 + 6,
 115         k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
 116 };
 117 #endif
 118
 119 #ifdef CONFIG_X86_64
 120
 121 extern char __vsyscall_0;
 122 static inline unsigned char** find_nop_table(void)
 123 {
 124         return k8_nops;
 125 }
 126
 127 #else /* CONFIG_X86_64 */
 128
 129 static struct nop {
 130         int cpuid;
 131         unsigned char **noptable;
 132 } noptypes[] = {
 133         { X86_FEATURE_K8, k8_nops },
 134         { X86_FEATURE_K7, k7_nops },
 135         { -1, NULL }
 136 };
 137
 138 static unsigned char** find_nop_table(void)
 139 {
 140         unsigned char **noptable = intel_nops;
 141         int i;
 142
 143         for (i = 0; noptypes[i].cpuid >= 0; i++) {
 144                 if (boot_cpu_has(noptypes[i].cpuid)) {
 145                         noptable = noptypes[i].noptable;
 146                         break;
 147                 }
 148         }
 149         return noptable;
 150 }
 151
 152 #endif /* CONFIG_X86_64 */
 153
 154 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
 155 static void add_nops(void *insns, unsigned int len)
 156 {
 157         unsigned char **noptable = find_nop_table();
 158
 159         while (len > 0) {
 160                 unsigned int noplen = len;
 161                 if (noplen > ASM_NOP_MAX)
 162                         noplen = ASM_NOP_MAX;
 163                 memcpy(insns, noptable[noplen], noplen);
 164                 insns += noplen;
 165                 len -= noplen;
 166         }
 167 }
 168
 169 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 170 extern u8 *__smp_locks[], *__smp_locks_end[];
 171
 172 /* Replace instructions with better alternatives for this CPU type.
 173    This runs before SMP is initialized to avoid SMP problems with
 174    self modifying code. This implies that assymetric systems where
 175    APs have less capabilities than the boot processor are not handled.
 176    Tough. Make sure you disable such features by hand. */
 177
 178 void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
 179 {
 180         struct alt_instr *a;
 181         char insnbuf[MAX_PATCH_LEN];
 182
 183         DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
 184         for (a = start; a < end; a++) {
 185                 u8 *instr = a->instr;
 186                 BUG_ON(a->replacementlen > a->instrlen);
 187                 BUG_ON(a->instrlen > sizeof(insnbuf));
 188                 if (!boot_cpu_has(a->cpuid))
 189                         continue;
 190 #ifdef CONFIG_X86_64
 191                 /* vsyscall code is not mapped yet. resolve it manually. */
 192                 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
 193                         instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
 194                         DPRINTK("%s: vsyscall fixup: %p => %p\n",
 195                                 __FUNCTION__, a->instr, instr);
 196                 }
 197 #endif
 198                 memcpy(insnbuf, a->replacement, a->replacementlen);
 199                 add_nops(insnbuf + a->replacementlen,
 200                          a->instrlen - a->replacementlen);
 201                 text_poke(instr, insnbuf, a->instrlen);
 202         }
 203 }
 204
 205 #ifdef CONFIG_SMP
 206
 207 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
 208 {
 209         u8 **ptr;
 210
 211         for (ptr = start; ptr < end; ptr++) {
 212                 if (*ptr < text)
 213                         continue;
 214                 if (*ptr > text_end)
 215                         continue;
 216                 text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
 217         };
 218 }
 219
 220 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
 221 {
 222         u8 **ptr;
 223         char insn[1];
 224
 225         if (noreplace_smp)
 226                 return;
 227
 228         add_nops(insn, 1);
 229         for (ptr = start; ptr < end; ptr++) {
 230                 if (*ptr < text)
 231                         continue;
 232                 if (*ptr > text_end)
 233                         continue;
 234                 text_poke(*ptr, insn, 1);
 235         };
 236 }
 237
 238 struct smp_alt_module {
 239         /* what is this ??? */
 240         struct module   *mod;
 241         char            *name;
 242
 243         /* ptrs to lock prefixes */
 244         u8              **locks;
 245         u8              **locks_end;
 246
 247         /* .text segment, needed to avoid patching init code ;) */
 248         u8              *text;
 249         u8              *text_end;
 250
 251         struct list_head next;
 252 };
 253 static LIST_HEAD(smp_alt_modules);
 254 static DEFINE_SPINLOCK(smp_alt);
 255
 256 void alternatives_smp_module_add(struct module *mod, char *name,
 257                                  void *locks, void *locks_end,
 258                                  void *text,  void *text_end)
 259 {
 260         struct smp_alt_module *smp;
 261         unsigned long flags;
 262
 263         if (noreplace_smp)
 264                 return;
 265
 266         if (smp_alt_once) {
 267                 if (boot_cpu_has(X86_FEATURE_UP))
 268                         alternatives_smp_unlock(locks, locks_end,
 269                                                 text, text_end);
 270                 return;
 271         }
 272
 273         smp = kzalloc(sizeof(*smp), GFP_KERNEL);
 274         if (NULL == smp)
 275                 return; /* we'll run the (safe but slow) SMP code then ... */
 276
 277         smp->mod        = mod;
 278         smp->name       = name;
 279         smp->locks      = locks;
 280         smp->locks_end  = locks_end;
 281         smp->text       = text;
 282         smp->text_end   = text_end;
 283         DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
 284                 __FUNCTION__, smp->locks, smp->locks_end,
 285                 smp->text, smp->text_end, smp->name);
 286
 287         spin_lock_irqsave(&smp_alt, flags);
 288         list_add_tail(&smp->next, &smp_alt_modules);
 289         if (boot_cpu_has(X86_FEATURE_UP))
 290                 alternatives_smp_unlock(smp->locks, smp->locks_end,
 291                                         smp->text, smp->text_end);
 292         spin_unlock_irqrestore(&smp_alt, flags);
 293 }
 294
 295 void alternatives_smp_module_del(struct module *mod)
 296 {
 297         struct smp_alt_module *item;
 298         unsigned long flags;
 299
 300         if (smp_alt_once || noreplace_smp)
 301                 return;
 302
 303         spin_lock_irqsave(&smp_alt, flags);
 304         list_for_each_entry(item, &smp_alt_modules, next) {
 305                 if (mod != item->mod)
 306                         continue;
 307                 list_del(&item->next);
 308                 spin_unlock_irqrestore(&smp_alt, flags);
 309                 DPRINTK("%s: %s\n", __FUNCTION__, item->name);
 310                 kfree(item);
 311                 return;
 312         }
 313         spin_unlock_irqrestore(&smp_alt, flags);
 314 }
 315
 316 void alternatives_smp_switch(int smp)
 317 {
 318         struct smp_alt_module *mod;
 319         unsigned long flags;
 320
 321 #ifdef CONFIG_LOCKDEP
 322         /*
 323          * A not yet fixed binutils section handling bug prevents
 324          * alternatives-replacement from working reliably, so turn
 325          * it off:
 326          */
 327         printk("lockdep: not fixing up alternatives.\n");
 328         return;
 329 #endif
 330
 331         if (noreplace_smp || smp_alt_once)
 332                 return;
 333         BUG_ON(!smp && (num_online_cpus() > 1));
 334
 335         spin_lock_irqsave(&smp_alt, flags);
 336         if (smp) {
 337                 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
 338                 clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
 339                 clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
 340                 list_for_each_entry(mod, &smp_alt_modules, next)
 341                         alternatives_smp_lock(mod->locks, mod->locks_end,
 342                                               mod->text, mod->text_end);
 343         } else {
 344                 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
 345                 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
 346                 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
 347                 list_for_each_entry(mod, &smp_alt_modules, next)
 348                         alternatives_smp_unlock(mod->locks, mod->locks_end,
 349                                                 mod->text, mod->text_end);
 350         }
 351         spin_unlock_irqrestore(&smp_alt, flags);
 352 }
 353
 354 #endif
 355
 356 #ifdef CONFIG_PARAVIRT
 357 void apply_paravirt(struct paravirt_patch_site *start,
 358                     struct paravirt_patch_site *end)
 359 {
 360         struct paravirt_patch_site *p;
 361         char insnbuf[MAX_PATCH_LEN];
 362
 363         if (noreplace_paravirt)
 364                 return;
 365
 366         for (p = start; p < end; p++) {
 367                 unsigned int used;
 368
 369                 BUG_ON(p->len > MAX_PATCH_LEN);
 370                 /* prep the buffer with the original instructions */
 371                 memcpy(insnbuf, p->instr, p->len);
 372                 used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
 373                                           (unsigned long)p->instr, p->len);
 374
 375                 BUG_ON(used > p->len);
 376
 377                 /* Pad the rest with nops */
 378                 add_nops(insnbuf + used, p->len - used);
 379                 text_poke(p->instr, insnbuf, p->len);
 380         }
 381 }
 382 extern struct paravirt_patch_site __start_parainstructions[],
 383         __stop_parainstructions[];
 384 #endif  /* CONFIG_PARAVIRT */
 385
 386 void __init alternative_instructions(void)
 387 {
 388         unsigned long flags;
 389
 390         /* The patching is not fully atomic, so try to avoid local interruptions
 391            that might execute the to be patched code.
 392            Other CPUs are not running. */
 393         stop_nmi();
 394 #ifdef CONFIG_X86_MCE
 395         stop_mce();
 396 #endif
 397
 398         local_irq_save(flags);
 399         apply_alternatives(__alt_instructions, __alt_instructions_end);
 400
 401         /* switch to patch-once-at-boottime-only mode and free the
 402          * tables in case we know the number of CPUs will never ever
 403          * change */
 404 #ifdef CONFIG_HOTPLUG_CPU
 405         if (num_possible_cpus() < 2)
 406                 smp_alt_once = 1;
 407 #endif
 408
 409 #ifdef CONFIG_SMP
 410         if (smp_alt_once) {
 411                 if (1 == num_possible_cpus()) {
 412                         printk(KERN_INFO "SMP alternatives: switching to UP code\n");
 413                         set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
 414                         set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
 415                         alternatives_smp_unlock(__smp_locks, __smp_locks_end,
 416                                                 _text, _etext);
 417                 }
 418         } else {
 419                 alternatives_smp_module_add(NULL, "core kernel",
 420                                             __smp_locks, __smp_locks_end,
 421                                             _text, _etext);
 422                 alternatives_smp_switch(0);
 423         }
 424 #endif
 425         apply_paravirt(__parainstructions, __parainstructions_end);
 426         local_irq_restore(flags);
 427
 428         if (smp_alt_once)
 429                 free_init_pages("SMP alternatives",
 430                                 (unsigned long)__smp_locks,
 431                                 (unsigned long)__smp_locks_end);
 432
 433         restart_nmi();
 434 #ifdef CONFIG_X86_MCE
 435         restart_mce();
 436 #endif
 437 }
 438
 439 /*
 440  * Warning:
 441  * When you use this code to patch more than one byte of an instruction
 442  * you need to make sure that other CPUs cannot execute this code in parallel.
 443  * Also no thread must be currently preempted in the middle of these instructions.
 444  * And on the local CPU you need to be protected again NMI or MCE handlers
 445  * seeing an inconsistent instruction while you patch.
 446  */
 447 void __kprobes text_poke(void *addr, unsigned char *opcode, int len)
 448 {
 449         memcpy(addr, opcode, len);
 450         sync_core();
 451         /* Could also do a CLFLUSH here to speed up CPU recovery; but
 452            that causes hangs on some VIA CPUs. */
 453 }