arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17
  18 #include <asm/spec-ctrl.h>
  19 #include <asm/cmdline.h>
  20 #include <asm/bugs.h>
  21 #include <asm/processor.h>
  22 #include <asm/processor-flags.h>
  23 #include <asm/fpu/internal.h>
  24 #include <asm/msr.h>
  25 #include <asm/vmx.h>
  26 #include <asm/paravirt.h>
  27 #include <asm/alternative.h>
  28 #include <asm/pgtable.h>
  29 #include <asm/set_memory.h>
  30 #include <asm/intel-family.h>
  31 #include <asm/e820/api.h>
  32
  33 static void __init spectre_v2_select_mitigation(void);
  34 static void __init ssb_select_mitigation(void);
  35 static void __init l1tf_select_mitigation(void);
  36
  37 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  38 u64 x86_spec_ctrl_base;
  39 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  40 static DEFINE_MUTEX(spec_ctrl_mutex);
  41
  42 /*
  43  * The vendor and possibly platform specific bits which can be modified in
  44  * x86_spec_ctrl_base.
  45  */
  46 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  47
  48 /*
  49  * AMD specific MSR info for Speculative Store Bypass control.
  50  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  51  */
  52 u64 __ro_after_init x86_amd_ls_cfg_base;
  53 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  54
  55 void __init check_bugs(void)
  56 {
  57         identify_boot_cpu();
  58
  59         /*
  60          * identify_boot_cpu() initialized SMT support information, let the
  61          * core code know.
  62          */
  63         cpu_smt_check_topology_early();
  64
  65         if (!IS_ENABLED(CONFIG_SMP)) {
  66                 pr_info("CPU: ");
  67                 print_cpu_info(&boot_cpu_data);
  68         }
  69
  70         /*
  71          * Read the SPEC_CTRL MSR to account for reserved bits which may
  72          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  73          * init code as it is not enumerated and depends on the family.
  74          */
  75         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  76                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  77
  78         /* Allow STIBP in MSR_SPEC_CTRL if supported */
  79         if (boot_cpu_has(X86_FEATURE_STIBP))
  80                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
  81
  82         /* Select the proper spectre mitigation before patching alternatives */
  83         spectre_v2_select_mitigation();
  84
  85         /*
  86          * Select proper mitigation for any exposure to the Speculative Store
  87          * Bypass vulnerability.
  88          */
  89         ssb_select_mitigation();
  90
  91         l1tf_select_mitigation();
  92
  93 #ifdef CONFIG_X86_32
  94         /*
  95          * Check whether we are able to run this kernel safely on SMP.
  96          *
  97          * - i386 is no longer supported.
  98          * - In order to run on anything without a TSC, we need to be
  99          *   compiled for a i486.
 100          */
 101         if (boot_cpu_data.x86 < 4)
 102                 panic("Kernel requires i486+ for 'invlpg' and other features");
 103
 104         init_utsname()->machine[1] =
 105                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 106         alternative_instructions();
 107
 108         fpu__init_check_bugs();
 109 #else /* CONFIG_X86_64 */
 110         alternative_instructions();
 111
 112         /*
 113          * Make sure the first 2MB area is not mapped by huge pages
 114          * There are typically fixed size MTRRs in there and overlapping
 115          * MTRRs into large pages causes slow downs.
 116          *
 117          * Right now we don't do that with gbpages because there seems
 118          * very little benefit for that case.
 119          */
 120         if (!direct_gbpages)
 121                 set_memory_4k((unsigned long)__va(0), 1);
 122 #endif
 123 }
 124
 125 /* The kernel command line selection */
 126 enum spectre_v2_mitigation_cmd {
 127         SPECTRE_V2_CMD_NONE,
 128         SPECTRE_V2_CMD_AUTO,
 129         SPECTRE_V2_CMD_FORCE,
 130         SPECTRE_V2_CMD_RETPOLINE,
 131         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 132         SPECTRE_V2_CMD_RETPOLINE_AMD,
 133 };
 134
 135 static const char *spectre_v2_strings[] = {
 136         [SPECTRE_V2_NONE]                       = "Vulnerable",
 137         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 138         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 139         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 140         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 141         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 142 };
 143
 144 #undef pr_fmt
 145 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 146
 147 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 148         SPECTRE_V2_NONE;
 149
 150 void
 151 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 152 {
 153         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 154         struct thread_info *ti = current_thread_info();
 155
 156         /* Is MSR_SPEC_CTRL implemented ? */
 157         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 158                 /*
 159                  * Restrict guest_spec_ctrl to supported values. Clear the
 160                  * modifiable bits in the host base value and or the
 161                  * modifiable bits from the guest value.
 162                  */
 163                 guestval = hostval & ~x86_spec_ctrl_mask;
 164                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 165
 166                 /* SSBD controlled in MSR_SPEC_CTRL */
 167                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 168                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 169                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 170
 171                 if (hostval != guestval) {
 172                         msrval = setguest ? guestval : hostval;
 173                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 174                 }
 175         }
 176
 177         /*
 178          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 179          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 180          */
 181         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 182             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 183                 return;
 184
 185         /*
 186          * If the host has SSBD mitigation enabled, force it in the host's
 187          * virtual MSR value. If its not permanently enabled, evaluate
 188          * current's TIF_SSBD thread flag.
 189          */
 190         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 191                 hostval = SPEC_CTRL_SSBD;
 192         else
 193                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 194
 195         /* Sanitize the guest value */
 196         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 197
 198         if (hostval != guestval) {
 199                 unsigned long tif;
 200
 201                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 202                                  ssbd_spec_ctrl_to_tif(hostval);
 203
 204                 speculative_store_bypass_update(tif);
 205         }
 206 }
 207 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 208
 209 static void x86_amd_ssb_disable(void)
 210 {
 211         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 212
 213         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 214                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 215         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 216                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 217 }
 218
 219 #ifdef RETPOLINE
 220 static bool spectre_v2_bad_module;
 221
 222 bool retpoline_module_ok(bool has_retpoline)
 223 {
 224         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 225                 return true;
 226
 227         pr_err("System may be vulnerable to spectre v2\n");
 228         spectre_v2_bad_module = true;
 229         return false;
 230 }
 231
 232 static inline const char *spectre_v2_module_string(void)
 233 {
 234         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 235 }
 236 #else
 237 static inline const char *spectre_v2_module_string(void) { return ""; }
 238 #endif
 239
 240 static void __init spec2_print_if_insecure(const char *reason)
 241 {
 242         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 243                 pr_info("%s selected on command line.\n", reason);
 244 }
 245
 246 static void __init spec2_print_if_secure(const char *reason)
 247 {
 248         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 249                 pr_info("%s selected on command line.\n", reason);
 250 }
 251
 252 static inline bool retp_compiler(void)
 253 {
 254         return __is_defined(RETPOLINE);
 255 }
 256
 257 static inline bool match_option(const char *arg, int arglen, const char *opt)
 258 {
 259         int len = strlen(opt);
 260
 261         return len == arglen && !strncmp(arg, opt, len);
 262 }
 263
 264 static const struct {
 265         const char *option;
 266         enum spectre_v2_mitigation_cmd cmd;
 267         bool secure;
 268 } mitigation_options[] = {
 269         { "off",               SPECTRE_V2_CMD_NONE,              false },
 270         { "on",                SPECTRE_V2_CMD_FORCE,             true },
 271         { "retpoline",         SPECTRE_V2_CMD_RETPOLINE,         false },
 272         { "retpoline,amd",     SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 273         { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 274         { "auto",              SPECTRE_V2_CMD_AUTO,              false },
 275 };
 276
 277 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 278 {
 279         char arg[20];
 280         int ret, i;
 281         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 282
 283         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 284                 return SPECTRE_V2_CMD_NONE;
 285         else {
 286                 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 287                 if (ret < 0)
 288                         return SPECTRE_V2_CMD_AUTO;
 289
 290                 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 291                         if (!match_option(arg, ret, mitigation_options[i].option))
 292                                 continue;
 293                         cmd = mitigation_options[i].cmd;
 294                         break;
 295                 }
 296
 297                 if (i >= ARRAY_SIZE(mitigation_options)) {
 298                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 299                         return SPECTRE_V2_CMD_AUTO;
 300                 }
 301         }
 302
 303         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 304              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 305              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 306             !IS_ENABLED(CONFIG_RETPOLINE)) {
 307                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 308                 return SPECTRE_V2_CMD_AUTO;
 309         }
 310
 311         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 312             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 313                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 314                 return SPECTRE_V2_CMD_AUTO;
 315         }
 316
 317         if (mitigation_options[i].secure)
 318                 spec2_print_if_secure(mitigation_options[i].option);
 319         else
 320                 spec2_print_if_insecure(mitigation_options[i].option);
 321
 322         return cmd;
 323 }
 324
 325 static bool stibp_needed(void)
 326 {
 327         if (spectre_v2_enabled == SPECTRE_V2_NONE)
 328                 return false;
 329
 330         if (!boot_cpu_has(X86_FEATURE_STIBP))
 331                 return false;
 332
 333         return true;
 334 }
 335
 336 static void update_stibp_msr(void *info)
 337 {
 338         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 339 }
 340
 341 void arch_smt_update(void)
 342 {
 343         u64 mask;
 344
 345         if (!stibp_needed())
 346                 return;
 347
 348         mutex_lock(&spec_ctrl_mutex);
 349         mask = x86_spec_ctrl_base;
 350         if (cpu_smt_control == CPU_SMT_ENABLED)
 351                 mask |= SPEC_CTRL_STIBP;
 352         else
 353                 mask &= ~SPEC_CTRL_STIBP;
 354
 355         if (mask != x86_spec_ctrl_base) {
 356                 pr_info("Spectre v2 cross-process SMT mitigation: %s STIBP\n",
 357                                 cpu_smt_control == CPU_SMT_ENABLED ?
 358                                 "Enabling" : "Disabling");
 359                 x86_spec_ctrl_base = mask;
 360                 on_each_cpu(update_stibp_msr, NULL, 1);
 361         }
 362         mutex_unlock(&spec_ctrl_mutex);
 363 }
 364
 365 static void __init spectre_v2_select_mitigation(void)
 366 {
 367         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 368         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 369
 370         /*
 371          * If the CPU is not affected and the command line mode is NONE or AUTO
 372          * then nothing to do.
 373          */
 374         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 375             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 376                 return;
 377
 378         switch (cmd) {
 379         case SPECTRE_V2_CMD_NONE:
 380                 return;
 381
 382         case SPECTRE_V2_CMD_FORCE:
 383         case SPECTRE_V2_CMD_AUTO:
 384                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 385                         mode = SPECTRE_V2_IBRS_ENHANCED;
 386                         /* Force it so VMEXIT will restore correctly */
 387                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 388                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 389                         goto specv2_set_mode;
 390                 }
 391                 if (IS_ENABLED(CONFIG_RETPOLINE))
 392                         goto retpoline_auto;
 393                 break;
 394         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 395                 if (IS_ENABLED(CONFIG_RETPOLINE))
 396                         goto retpoline_amd;
 397                 break;
 398         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 399                 if (IS_ENABLED(CONFIG_RETPOLINE))
 400                         goto retpoline_generic;
 401                 break;
 402         case SPECTRE_V2_CMD_RETPOLINE:
 403                 if (IS_ENABLED(CONFIG_RETPOLINE))
 404                         goto retpoline_auto;
 405                 break;
 406         }
 407         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 408         return;
 409
 410 retpoline_auto:
 411         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 412         retpoline_amd:
 413                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 414                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 415                         goto retpoline_generic;
 416                 }
 417                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 418                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 419                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 420                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 421         } else {
 422         retpoline_generic:
 423                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 424                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 425                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 426         }
 427
 428 specv2_set_mode:
 429         spectre_v2_enabled = mode;
 430         pr_info("%s\n", spectre_v2_strings[mode]);
 431
 432         /*
 433          * If spectre v2 protection has been enabled, unconditionally fill
 434          * RSB during a context switch; this protects against two independent
 435          * issues:
 436          *
 437          *      - RSB underflow (and switch to BTB) on Skylake+
 438          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 439          */
 440         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 441         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 442
 443         /* Initialize Indirect Branch Prediction Barrier if supported */
 444         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 445                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 446                 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
 447         }
 448
 449         /*
 450          * Retpoline means the kernel is safe because it has no indirect
 451          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 452          * speculation around firmware calls only when Enhanced IBRS isn't
 453          * supported.
 454          *
 455          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 456          * the user might select retpoline on the kernel command line and if
 457          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 458          * enable IBRS around firmware calls.
 459          */
 460         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 461                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 462                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 463         }
 464
 465         /* Enable STIBP if appropriate */
 466         arch_smt_update();
 467 }
 468
 469 #undef pr_fmt
 470 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 471
 472 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 473
 474 /* The kernel command line selection */
 475 enum ssb_mitigation_cmd {
 476         SPEC_STORE_BYPASS_CMD_NONE,
 477         SPEC_STORE_BYPASS_CMD_AUTO,
 478         SPEC_STORE_BYPASS_CMD_ON,
 479         SPEC_STORE_BYPASS_CMD_PRCTL,
 480         SPEC_STORE_BYPASS_CMD_SECCOMP,
 481 };
 482
 483 static const char *ssb_strings[] = {
 484         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 485         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 486         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 487         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 488 };
 489
 490 static const struct {
 491         const char *option;
 492         enum ssb_mitigation_cmd cmd;
 493 } ssb_mitigation_options[] = {
 494         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 495         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 496         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 497         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 498         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 499 };
 500
 501 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 502 {
 503         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 504         char arg[20];
 505         int ret, i;
 506
 507         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 508                 return SPEC_STORE_BYPASS_CMD_NONE;
 509         } else {
 510                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 511                                           arg, sizeof(arg));
 512                 if (ret < 0)
 513                         return SPEC_STORE_BYPASS_CMD_AUTO;
 514
 515                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 516                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 517                                 continue;
 518
 519                         cmd = ssb_mitigation_options[i].cmd;
 520                         break;
 521                 }
 522
 523                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 524                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 525                         return SPEC_STORE_BYPASS_CMD_AUTO;
 526                 }
 527         }
 528
 529         return cmd;
 530 }
 531
 532 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 533 {
 534         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 535         enum ssb_mitigation_cmd cmd;
 536
 537         if (!boot_cpu_has(X86_FEATURE_SSBD))
 538                 return mode;
 539
 540         cmd = ssb_parse_cmdline();
 541         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 542             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 543              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 544                 return mode;
 545
 546         switch (cmd) {
 547         case SPEC_STORE_BYPASS_CMD_AUTO:
 548         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 549                 /*
 550                  * Choose prctl+seccomp as the default mode if seccomp is
 551                  * enabled.
 552                  */
 553                 if (IS_ENABLED(CONFIG_SECCOMP))
 554                         mode = SPEC_STORE_BYPASS_SECCOMP;
 555                 else
 556                         mode = SPEC_STORE_BYPASS_PRCTL;
 557                 break;
 558         case SPEC_STORE_BYPASS_CMD_ON:
 559                 mode = SPEC_STORE_BYPASS_DISABLE;
 560                 break;
 561         case SPEC_STORE_BYPASS_CMD_PRCTL:
 562                 mode = SPEC_STORE_BYPASS_PRCTL;
 563                 break;
 564         case SPEC_STORE_BYPASS_CMD_NONE:
 565                 break;
 566         }
 567
 568         /*
 569          * We have three CPU feature flags that are in play here:
 570          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 571          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 572          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 573          */
 574         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 575                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 576                 /*
 577                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
 578                  * use a completely different MSR and bit dependent on family.
 579                  */
 580                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
 581                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
 582                         x86_amd_ssb_disable();
 583                 } else {
 584                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 585                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
 586                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 587                 }
 588         }
 589
 590         return mode;
 591 }
 592
 593 static void ssb_select_mitigation(void)
 594 {
 595         ssb_mode = __ssb_select_mitigation();
 596
 597         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 598                 pr_info("%s\n", ssb_strings[ssb_mode]);
 599 }
 600
 601 #undef pr_fmt
 602 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 603
 604 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 605 {
 606         bool update;
 607
 608         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 609             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 610                 return -ENXIO;
 611
 612         switch (ctrl) {
 613         case PR_SPEC_ENABLE:
 614                 /* If speculation is force disabled, enable is not allowed */
 615                 if (task_spec_ssb_force_disable(task))
 616                         return -EPERM;
 617                 task_clear_spec_ssb_disable(task);
 618                 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
 619                 break;
 620         case PR_SPEC_DISABLE:
 621                 task_set_spec_ssb_disable(task);
 622                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 623                 break;
 624         case PR_SPEC_FORCE_DISABLE:
 625                 task_set_spec_ssb_disable(task);
 626                 task_set_spec_ssb_force_disable(task);
 627                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 628                 break;
 629         default:
 630                 return -ERANGE;
 631         }
 632
 633         /*
 634          * If being set on non-current task, delay setting the CPU
 635          * mitigation until it is next scheduled.
 636          */
 637         if (task == current && update)
 638                 speculative_store_bypass_update_current();
 639
 640         return 0;
 641 }
 642
 643 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 644                              unsigned long ctrl)
 645 {
 646         switch (which) {
 647         case PR_SPEC_STORE_BYPASS:
 648                 return ssb_prctl_set(task, ctrl);
 649         default:
 650                 return -ENODEV;
 651         }
 652 }
 653
 654 #ifdef CONFIG_SECCOMP
 655 void arch_seccomp_spec_mitigate(struct task_struct *task)
 656 {
 657         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 658                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 659 }
 660 #endif
 661
 662 static int ssb_prctl_get(struct task_struct *task)
 663 {
 664         switch (ssb_mode) {
 665         case SPEC_STORE_BYPASS_DISABLE:
 666                 return PR_SPEC_DISABLE;
 667         case SPEC_STORE_BYPASS_SECCOMP:
 668         case SPEC_STORE_BYPASS_PRCTL:
 669                 if (task_spec_ssb_force_disable(task))
 670                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 671                 if (task_spec_ssb_disable(task))
 672                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 673                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 674         default:
 675                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 676                         return PR_SPEC_ENABLE;
 677                 return PR_SPEC_NOT_AFFECTED;
 678         }
 679 }
 680
 681 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 682 {
 683         switch (which) {
 684         case PR_SPEC_STORE_BYPASS:
 685                 return ssb_prctl_get(task);
 686         default:
 687                 return -ENODEV;
 688         }
 689 }
 690
 691 void x86_spec_ctrl_setup_ap(void)
 692 {
 693         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
 694                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 695
 696         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
 697                 x86_amd_ssb_disable();
 698 }
 699
 700 #undef pr_fmt
 701 #define pr_fmt(fmt)     "L1TF: " fmt
 702
 703 /* Default mitigation for L1TF-affected CPUs */
 704 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
 705 #if IS_ENABLED(CONFIG_KVM_INTEL)
 706 EXPORT_SYMBOL_GPL(l1tf_mitigation);
 707 #endif
 708 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
 709 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
 710
 711 /*
 712  * These CPUs all support 44bits physical address space internally in the
 713  * cache but CPUID can report a smaller number of physical address bits.
 714  *
 715  * The L1TF mitigation uses the top most address bit for the inversion of
 716  * non present PTEs. When the installed memory reaches into the top most
 717  * address bit due to memory holes, which has been observed on machines
 718  * which report 36bits physical address bits and have 32G RAM installed,
 719  * then the mitigation range check in l1tf_select_mitigation() triggers.
 720  * This is a false positive because the mitigation is still possible due to
 721  * the fact that the cache uses 44bit internally. Use the cache bits
 722  * instead of the reported physical bits and adjust them on the affected
 723  * machines to 44bit if the reported bits are less than 44.
 724  */
 725 static void override_cache_bits(struct cpuinfo_x86 *c)
 726 {
 727         if (c->x86 != 6)
 728                 return;
 729
 730         switch (c->x86_model) {
 731         case INTEL_FAM6_NEHALEM:
 732         case INTEL_FAM6_WESTMERE:
 733         case INTEL_FAM6_SANDYBRIDGE:
 734         case INTEL_FAM6_IVYBRIDGE:
 735         case INTEL_FAM6_HASWELL_CORE:
 736         case INTEL_FAM6_HASWELL_ULT:
 737         case INTEL_FAM6_HASWELL_GT3E:
 738         case INTEL_FAM6_BROADWELL_CORE:
 739         case INTEL_FAM6_BROADWELL_GT3E:
 740         case INTEL_FAM6_SKYLAKE_MOBILE:
 741         case INTEL_FAM6_SKYLAKE_DESKTOP:
 742         case INTEL_FAM6_KABYLAKE_MOBILE:
 743         case INTEL_FAM6_KABYLAKE_DESKTOP:
 744                 if (c->x86_cache_bits < 44)
 745                         c->x86_cache_bits = 44;
 746                 break;
 747         }
 748 }
 749
 750 static void __init l1tf_select_mitigation(void)
 751 {
 752         u64 half_pa;
 753
 754         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 755                 return;
 756
 757         override_cache_bits(&boot_cpu_data);
 758
 759         switch (l1tf_mitigation) {
 760         case L1TF_MITIGATION_OFF:
 761         case L1TF_MITIGATION_FLUSH_NOWARN:
 762         case L1TF_MITIGATION_FLUSH:
 763                 break;
 764         case L1TF_MITIGATION_FLUSH_NOSMT:
 765         case L1TF_MITIGATION_FULL:
 766                 cpu_smt_disable(false);
 767                 break;
 768         case L1TF_MITIGATION_FULL_FORCE:
 769                 cpu_smt_disable(true);
 770                 break;
 771         }
 772
 773 #if CONFIG_PGTABLE_LEVELS == 2
 774         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
 775         return;
 776 #endif
 777
 778         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
 779         if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
 780                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
 781                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
 782                                 half_pa);
 783                 pr_info("However, doing so will make a part of your RAM unusable.\n");
 784                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
 785                 return;
 786         }
 787
 788         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
 789 }
 790
 791 static int __init l1tf_cmdline(char *str)
 792 {
 793         if (!boot_cpu_has_bug(X86_BUG_L1TF))
 794                 return 0;
 795
 796         if (!str)
 797                 return -EINVAL;
 798
 799         if (!strcmp(str, "off"))
 800                 l1tf_mitigation = L1TF_MITIGATION_OFF;
 801         else if (!strcmp(str, "flush,nowarn"))
 802                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
 803         else if (!strcmp(str, "flush"))
 804                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
 805         else if (!strcmp(str, "flush,nosmt"))
 806                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
 807         else if (!strcmp(str, "full"))
 808                 l1tf_mitigation = L1TF_MITIGATION_FULL;
 809         else if (!strcmp(str, "full,force"))
 810                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
 811
 812         return 0;
 813 }
 814 early_param("l1tf", l1tf_cmdline);
 815
 816 #undef pr_fmt
 817
 818 #ifdef CONFIG_SYSFS
 819
 820 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
 821
 822 #if IS_ENABLED(CONFIG_KVM_INTEL)
 823 static const char *l1tf_vmx_states[] = {
 824         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
 825         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
 826         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
 827         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
 828         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
 829         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
 830 };
 831
 832 static ssize_t l1tf_show_state(char *buf)
 833 {
 834         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
 835                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 836
 837         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
 838             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
 839              cpu_smt_control == CPU_SMT_ENABLED))
 840                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
 841                                l1tf_vmx_states[l1tf_vmx_mitigation]);
 842
 843         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
 844                        l1tf_vmx_states[l1tf_vmx_mitigation],
 845                        cpu_smt_control == CPU_SMT_ENABLED ? "vulnerable" : "disabled");
 846 }
 847 #else
 848 static ssize_t l1tf_show_state(char *buf)
 849 {
 850         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
 851 }
 852 #endif
 853
 854 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 855                                char *buf, unsigned int bug)
 856 {
 857         int ret;
 858
 859         if (!boot_cpu_has_bug(bug))
 860                 return sprintf(buf, "Not affected\n");
 861
 862         switch (bug) {
 863         case X86_BUG_CPU_MELTDOWN:
 864                 if (boot_cpu_has(X86_FEATURE_PTI))
 865                         return sprintf(buf, "Mitigation: PTI\n");
 866
 867                 break;
 868
 869         case X86_BUG_SPECTRE_V1:
 870                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 871
 872         case X86_BUG_SPECTRE_V2:
 873                 ret = sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
 874                                boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
 875                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
 876                                (x86_spec_ctrl_base & SPEC_CTRL_STIBP) ? ", STIBP" : "",
 877                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
 878                                spectre_v2_module_string());
 879                 return ret;
 880
 881         case X86_BUG_SPEC_STORE_BYPASS:
 882                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
 883
 884         case X86_BUG_L1TF:
 885                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
 886                         return l1tf_show_state(buf);
 887                 break;
 888         default:
 889                 break;
 890         }
 891
 892         return sprintf(buf, "Vulnerable\n");
 893 }
 894
 895 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
 896 {
 897         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 898 }
 899
 900 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
 901 {
 902         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 903 }
 904
 905 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
 906 {
 907         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 908 }
 909
 910 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
 911 {
 912         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
 913 }
 914
 915 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
 916 {
 917         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
 918 }
 919 #endif