projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: Reorder ioctls in kvm.h
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / x86 / kvm / vmx.c
CommitLineData
6aa8b732
AK		 1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
85f455f7 18#include "irq.h"
1d737c8a 19#include "mmu.h"
e495606d 20
edf88417 21#include <linux/kvm_host.h>
6aa8b732 22#include <linux/module.h>
9d8f549d 23#include <linux/kernel.h>
6aa8b732
AK		 24#include <linux/mm.h>
25#include <linux/highmem.h>
e8edc6e0 26#include <linux/sched.h>
c7addb90 27#include <linux/moduleparam.h>
5fdbf976 28#include "kvm_cache_regs.h"
35920a35 29#include "x86.h"
e495606d 30
6aa8b732 31#include <asm/io.h>
3b3be0d1 32#include <asm/desc.h>
13673a90 33#include <asm/vmx.h>
6210e37b 34#include <asm/virtext.h>
a0861c02 35#include <asm/mce.h>
6aa8b732 36
4ecac3fd
AK		 37#define __ex(x) __kvm_handle_fault_on_reboot(x)
38
6aa8b732
AK		 39MODULE_AUTHOR("Qumranet");
40MODULE_LICENSE("GPL");
41
4462d21a 42static int __read_mostly bypass_guest_pf = 1;
c1f8bc04 43module_param(bypass_guest_pf, bool, S_IRUGO);
c7addb90 44
4462d21a 45static int __read_mostly enable_vpid = 1;
736caefe 46module_param_named(vpid, enable_vpid, bool, 0444);
2384d2b3 47
4462d21a 48static int __read_mostly flexpriority_enabled = 1;
736caefe 49module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
4c9fc8ef 50
4462d21a 51static int __read_mostly enable_ept = 1;
736caefe 52module_param_named(ept, enable_ept, bool, S_IRUGO);
d56f546d 53
3a624e29
NK		 54static int __read_mostly enable_unrestricted_guest = 1;
55module_param_named(unrestricted_guest,
56 enable_unrestricted_guest, bool, S_IRUGO);
57
4462d21a 58static int __read_mostly emulate_invalid_guest_state = 0;
c1f8bc04 59module_param(emulate_invalid_guest_state, bool, S_IRUGO);
04fa4d32 60
a2fa3e9f
GH		 61struct vmcs {
62 u32 revision_id;
63 u32 abort;
64 char data[0];
65};
66
67struct vcpu_vmx {
fb3f0f51 68 struct kvm_vcpu vcpu;
543e4243 69 struct list_head local_vcpus_link;
313dbd49 70 unsigned long host_rsp;
a2fa3e9f 71 int launched;
29bd8a78 72 u8 fail;
1155f76a 73 u32 idt_vectoring_info;
a2fa3e9f
GH		 74 struct kvm_msr_entry *guest_msrs;
75 struct kvm_msr_entry *host_msrs;
76 int nmsrs;
77 int save_nmsrs;
78 int msr_offset_efer;
79#ifdef CONFIG_X86_64
80 int msr_offset_kernel_gs_base;
81#endif
82 struct vmcs *vmcs;
83 struct {
84 int loaded;
85 u16 fs_sel, gs_sel, ldt_sel;
152d3f2f
LV		 86 int gs_ldt_reload_needed;
87 int fs_reload_needed;
51c6cf66 88 int guest_efer_loaded;
d77c26fc 89 } host_state;
9c8cba37
AK		 90 struct {
91 struct {
92 bool pending;
93 u8 vector;
94 unsigned rip;
95 } irq;
96 } rmode;
2384d2b3 97 int vpid;
04fa4d32 98 bool emulation_required;
8b3079a5 99 enum emulation_result invalid_state_emulation_result;
3b86cd99
JK		 100
101 /* Support for vnmi-less CPUs */
102 int soft_vnmi_blocked;
103 ktime_t entry_time;
104 s64 vnmi_blocked_time;
a0861c02 105 u32 exit_reason;
a2fa3e9f
GH		 106};
107
108static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
109{
fb3f0f51 110 return container_of(vcpu, struct vcpu_vmx, vcpu);
a2fa3e9f
GH		 111}
112
b7ebfb05 113static int init_rmode(struct kvm *kvm);
4e1096d2 114static u64 construct_eptp(unsigned long root_hpa);
75880a01 115
6aa8b732
AK		 116static DEFINE_PER_CPU(struct vmcs *, vmxarea);
117static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
543e4243 118static DEFINE_PER_CPU(struct list_head, vcpus_on_cpu);
6aa8b732 119
3e7c73e9
AK		 120static unsigned long *vmx_io_bitmap_a;
121static unsigned long *vmx_io_bitmap_b;
5897297b
AK		 122static unsigned long *vmx_msr_bitmap_legacy;
123static unsigned long *vmx_msr_bitmap_longmode;
fdef3ad1 124
2384d2b3
SY		 125static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
126static DEFINE_SPINLOCK(vmx_vpid_lock);
127
1c3d14fe 128static struct vmcs_config {
6aa8b732
AK		 129 int size;
130 int order;
131 u32 revision_id;
1c3d14fe
YS		 132 u32 pin_based_exec_ctrl;
133 u32 cpu_based_exec_ctrl;
f78e0e2e 134 u32 cpu_based_2nd_exec_ctrl;
1c3d14fe
YS		 135 u32 vmexit_ctrl;
136 u32 vmentry_ctrl;
137} vmcs_config;
6aa8b732 138
efff9e53 139static struct vmx_capability {
d56f546d
SY		 140 u32 ept;
141 u32 vpid;
142} vmx_capability;
143
6aa8b732
AK		 144#define VMX_SEGMENT_FIELD(seg) \
145 [VCPU_SREG_##seg] = { \
146 .selector = GUEST_##seg##_SELECTOR, \
147 .base = GUEST_##seg##_BASE, \
148 .limit = GUEST_##seg##_LIMIT, \
149 .ar_bytes = GUEST_##seg##_AR_BYTES, \
150 }
151
152static struct kvm_vmx_segment_field {
153 unsigned selector;
154 unsigned base;
155 unsigned limit;
156 unsigned ar_bytes;
157} kvm_vmx_segment_fields[] = {
158 VMX_SEGMENT_FIELD(CS),
159 VMX_SEGMENT_FIELD(DS),
160 VMX_SEGMENT_FIELD(ES),
161 VMX_SEGMENT_FIELD(FS),
162 VMX_SEGMENT_FIELD(GS),
163 VMX_SEGMENT_FIELD(SS),
164 VMX_SEGMENT_FIELD(TR),
165 VMX_SEGMENT_FIELD(LDTR),
166};
167
6de4f3ad
AK		 168static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
169
4d56c8a7
AK		 170/*
171 * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
172 * away by decrementing the array size.
173 */
6aa8b732 174static const u32 vmx_msr_index[] = {
05b3e0c2 175#ifdef CONFIG_X86_64
6aa8b732
AK		 176 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE,
177#endif
178 MSR_EFER, MSR_K6_STAR,
179};
9d8f549d 180#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
6aa8b732 181
a2fa3e9f
GH		 182static void load_msrs(struct kvm_msr_entry *e, int n)
183{
184 int i;
185
186 for (i = 0; i < n; ++i)
187 wrmsrl(e[i].index, e[i].data);
188}
189
190static void save_msrs(struct kvm_msr_entry *e, int n)
191{
192 int i;
193
194 for (i = 0; i < n; ++i)
195 rdmsrl(e[i].index, e[i].data);
196}
197
6aa8b732
AK		 198static inline int is_page_fault(u32 intr_info)
199{
200 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
201 INTR_INFO_VALID_MASK)) ==
8ab2d2e2 202 (INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
6aa8b732
AK		 203}
204
2ab455cc
AL		 205static inline int is_no_device(u32 intr_info)
206{
207 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
208 INTR_INFO_VALID_MASK)) ==
8ab2d2e2 209 (INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
2ab455cc
AL		 210}
211
7aa81cc0
AL		 212static inline int is_invalid_opcode(u32 intr_info)
213{
214 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
215 INTR_INFO_VALID_MASK)) ==
8ab2d2e2 216 (INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK);
7aa81cc0
AL		 217}
218
6aa8b732
AK		 219static inline int is_external_interrupt(u32 intr_info)
220{
221 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
222 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
223}
224
a0861c02
AK		 225static inline int is_machine_check(u32 intr_info)
226{
227 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
228 INTR_INFO_VALID_MASK)) ==
229 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
230}
231
25c5f225
SY		 232static inline int cpu_has_vmx_msr_bitmap(void)
233{
04547156 234 return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
25c5f225
SY		 235}
236
6e5d865c
YS		 237static inline int cpu_has_vmx_tpr_shadow(void)
238{
04547156 239 return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
6e5d865c
YS		 240}
241
242static inline int vm_need_tpr_shadow(struct kvm *kvm)
243{
04547156 244 return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
6e5d865c
YS		 245}
246
f78e0e2e
SY		 247static inline int cpu_has_secondary_exec_ctrls(void)
248{
04547156
SY		 249 return vmcs_config.cpu_based_exec_ctrl &
250 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
f78e0e2e
SY		 251}
252
774ead3a 253static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
f78e0e2e 254{
04547156
SY		 255 return vmcs_config.cpu_based_2nd_exec_ctrl &
256 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
257}
258
259static inline bool cpu_has_vmx_flexpriority(void)
260{
261 return cpu_has_vmx_tpr_shadow() &&
262 cpu_has_vmx_virtualize_apic_accesses();
f78e0e2e
SY		 263}
264
d56f546d
SY		 265static inline int cpu_has_vmx_invept_individual_addr(void)
266{
04547156 267 return !!(vmx_capability.ept & VMX_EPT_EXTENT_INDIVIDUAL_BIT);
d56f546d
SY		 268}
269
270static inline int cpu_has_vmx_invept_context(void)
271{
04547156 272 return !!(vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT);
d56f546d
SY		 273}
274
275static inline int cpu_has_vmx_invept_global(void)
276{
04547156 277 return !!(vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT);
d56f546d
SY		 278}
279
280static inline int cpu_has_vmx_ept(void)
281{
04547156
SY		 282 return vmcs_config.cpu_based_2nd_exec_ctrl &
283 SECONDARY_EXEC_ENABLE_EPT;
d56f546d
SY		 284}
285
3a624e29
NK		 286static inline int cpu_has_vmx_unrestricted_guest(void)
287{
288 return vmcs_config.cpu_based_2nd_exec_ctrl &
289 SECONDARY_EXEC_UNRESTRICTED_GUEST;
290}
291
f78e0e2e
SY		 292static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm)
293{
04547156
SY		 294 return flexpriority_enabled &&
295 (cpu_has_vmx_virtualize_apic_accesses()) &&
296 (irqchip_in_kernel(kvm));
f78e0e2e
SY		 297}
298
2384d2b3
SY		 299static inline int cpu_has_vmx_vpid(void)
300{
04547156
SY		 301 return vmcs_config.cpu_based_2nd_exec_ctrl &
302 SECONDARY_EXEC_ENABLE_VPID;
2384d2b3
SY		 303}
304
f08864b4
SY		 305static inline int cpu_has_virtual_nmis(void)
306{
307 return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
308}
309
04547156
SY		 310static inline bool report_flexpriority(void)
311{
312 return flexpriority_enabled;
313}
314
8b9cf98c 315static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
7725f0ba
AK		 316{
317 int i;
318
a2fa3e9f
GH		 319 for (i = 0; i < vmx->nmsrs; ++i)
320 if (vmx->guest_msrs[i].index == msr)
a75beee6
ED		 321 return i;
322 return -1;
323}
324
2384d2b3
SY		 325static inline void __invvpid(int ext, u16 vpid, gva_t gva)
326{
327 struct {
328 u64 vpid : 16;
329 u64 rsvd : 48;
330 u64 gva;
331 } operand = { vpid, 0, gva };
332
4ecac3fd 333 asm volatile (__ex(ASM_VMX_INVVPID)
2384d2b3
SY		 334 /* CF==1 or ZF==1 --> rc = -1 */
335 "; ja 1f ; ud2 ; 1:"
336 : : "a"(&operand), "c"(ext) : "cc", "memory");
337}
338
1439442c
SY		 339static inline void __invept(int ext, u64 eptp, gpa_t gpa)
340{
341 struct {
342 u64 eptp, gpa;
343 } operand = {eptp, gpa};
344
4ecac3fd 345 asm volatile (__ex(ASM_VMX_INVEPT)
1439442c
SY		 346 /* CF==1 or ZF==1 --> rc = -1 */
347 "; ja 1f ; ud2 ; 1:\n"
348 : : "a" (&operand), "c" (ext) : "cc", "memory");
349}
350
8b9cf98c 351static struct kvm_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
a75beee6
ED		 352{
353 int i;
354
8b9cf98c 355 i = __find_msr_index(vmx, msr);
a75beee6 356 if (i >= 0)
a2fa3e9f 357 return &vmx->guest_msrs[i];
8b6d44c7 358 return NULL;
7725f0ba
AK		 359}
360
6aa8b732
AK		 361static void vmcs_clear(struct vmcs *vmcs)
362{
363 u64 phys_addr = __pa(vmcs);
364 u8 error;
365
4ecac3fd 366 asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) "; setna %0"
6aa8b732
AK		 367 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
368 : "cc", "memory");
369 if (error)
370 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
371 vmcs, phys_addr);
372}
373
374static void __vcpu_clear(void *arg)
375{
8b9cf98c 376 struct vcpu_vmx *vmx = arg;
d3b2c338 377 int cpu = raw_smp_processor_id();
6aa8b732 378
8b9cf98c 379 if (vmx->vcpu.cpu == cpu)
a2fa3e9f
GH		 380 vmcs_clear(vmx->vmcs);
381 if (per_cpu(current_vmcs, cpu) == vmx->vmcs)
6aa8b732 382 per_cpu(current_vmcs, cpu) = NULL;
ad312c7c 383 rdtscll(vmx->vcpu.arch.host_tsc);
543e4243
AK		 384 list_del(&vmx->local_vcpus_link);
385 vmx->vcpu.cpu = -1;
386 vmx->launched = 0;
6aa8b732
AK		 387}
388
8b9cf98c 389static void vcpu_clear(struct vcpu_vmx *vmx)
8d0be2b3 390{
eae5ecb5
AK		 391 if (vmx->vcpu.cpu == -1)
392 return;
8691e5a8 393 smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 1);
8d0be2b3
AK		 394}
395
2384d2b3
SY		 396static inline void vpid_sync_vcpu_all(struct vcpu_vmx *vmx)
397{
398 if (vmx->vpid == 0)
399 return;
400
401 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vmx->vpid, 0);
402}
403
1439442c
SY		 404static inline void ept_sync_global(void)
405{
406 if (cpu_has_vmx_invept_global())
407 __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
408}
409
410static inline void ept_sync_context(u64 eptp)
411{
089d034e 412 if (enable_ept) {
1439442c
SY		 413 if (cpu_has_vmx_invept_context())
414 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
415 else
416 ept_sync_global();
417 }
418}
419
420static inline void ept_sync_individual_addr(u64 eptp, gpa_t gpa)
421{
089d034e 422 if (enable_ept) {
1439442c
SY		 423 if (cpu_has_vmx_invept_individual_addr())
424 __invept(VMX_EPT_EXTENT_INDIVIDUAL_ADDR,
425 eptp, gpa);
426 else
427 ept_sync_context(eptp);
428 }
429}
430
6aa8b732
AK		 431static unsigned long vmcs_readl(unsigned long field)
432{
433 unsigned long value;
434
4ecac3fd 435 asm volatile (__ex(ASM_VMX_VMREAD_RDX_RAX)
6aa8b732
AK		 436 : "=a"(value) : "d"(field) : "cc");
437 return value;
438}
439
440static u16 vmcs_read16(unsigned long field)
441{
442 return vmcs_readl(field);
443}
444
445static u32 vmcs_read32(unsigned long field)
446{
447 return vmcs_readl(field);
448}
449
450static u64 vmcs_read64(unsigned long field)
451{
05b3e0c2 452#ifdef CONFIG_X86_64
6aa8b732
AK		 453 return vmcs_readl(field);
454#else
455 return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
456#endif
457}
458
e52de1b8
AK		 459static noinline void vmwrite_error(unsigned long field, unsigned long value)
460{
461 printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
462 field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
463 dump_stack();
464}
465
6aa8b732
AK		 466static void vmcs_writel(unsigned long field, unsigned long value)
467{
468 u8 error;
469
4ecac3fd 470 asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) "; setna %0"
d77c26fc 471 : "=q"(error) : "a"(value), "d"(field) : "cc");
e52de1b8
AK		 472 if (unlikely(error))
473 vmwrite_error(field, value);
6aa8b732
AK		 474}
475
476static void vmcs_write16(unsigned long field, u16 value)
477{
478 vmcs_writel(field, value);
479}
480
481static void vmcs_write32(unsigned long field, u32 value)
482{
483 vmcs_writel(field, value);
484}
485
486static void vmcs_write64(unsigned long field, u64 value)
487{
6aa8b732 488 vmcs_writel(field, value);
7682f2d0 489#ifndef CONFIG_X86_64
6aa8b732
AK		 490 asm volatile ("");
491 vmcs_writel(field+1, value >> 32);
492#endif
493}
494
2ab455cc
AL		 495static void vmcs_clear_bits(unsigned long field, u32 mask)
496{
497 vmcs_writel(field, vmcs_readl(field) & ~mask);
498}
499
500static void vmcs_set_bits(unsigned long field, u32 mask)
501{
502 vmcs_writel(field, vmcs_readl(field) | mask);
503}
504
abd3f2d6
AK		 505static void update_exception_bitmap(struct kvm_vcpu *vcpu)
506{
507 u32 eb;
508
a0861c02 509 eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR);
abd3f2d6
AK		 510 if (!vcpu->fpu_active)
511 eb |= 1u << NM_VECTOR;
d0bfb940
JK		 512 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
513 if (vcpu->guest_debug &
514 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
515 eb |= 1u << DB_VECTOR;
516 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
517 eb |= 1u << BP_VECTOR;
518 }
56b237e3 519 if (vcpu->arch.rmode.vm86_active)
abd3f2d6 520 eb = ~0;
089d034e 521 if (enable_ept)
1439442c 522 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
abd3f2d6
AK		 523 vmcs_write32(EXCEPTION_BITMAP, eb);
524}
525
33ed6329
AK		 526static void reload_tss(void)
527{
33ed6329
AK		 528 /*
529 * VT restores TR but not its size. Useless.
530 */
531 struct descriptor_table gdt;
a5f61300 532 struct desc_struct *descs;
33ed6329 533
d6e88aec 534 kvm_get_gdt(&gdt);
33ed6329
AK		 535 descs = (void *)gdt.base;
536 descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
537 load_TR_desc();
33ed6329
AK		 538}
539
8b9cf98c 540static void load_transition_efer(struct vcpu_vmx *vmx)
2cc51560 541{
a2fa3e9f 542 int efer_offset = vmx->msr_offset_efer;
51c6cf66
AK		 543 u64 host_efer = vmx->host_msrs[efer_offset].data;
544 u64 guest_efer = vmx->guest_msrs[efer_offset].data;
545 u64 ignore_bits;
546
547 if (efer_offset < 0)
548 return;
549 /*
550 * NX is emulated; LMA and LME handled by hardware; SCE meaninless
551 * outside long mode
552 */
553 ignore_bits = EFER_NX | EFER_SCE;
554#ifdef CONFIG_X86_64
555 ignore_bits |= EFER_LMA | EFER_LME;
556 /* SCE is meaningful only in long mode on Intel */
557 if (guest_efer & EFER_LMA)
558 ignore_bits &= ~(u64)EFER_SCE;
559#endif
560 if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits))
561 return;
2cc51560 562
51c6cf66
AK		 563 vmx->host_state.guest_efer_loaded = 1;
564 guest_efer &= ~ignore_bits;
565 guest_efer |= host_efer & ignore_bits;
566 wrmsrl(MSR_EFER, guest_efer);
8b9cf98c 567 vmx->vcpu.stat.efer_reload++;
2cc51560
ED		 568}
569
51c6cf66
AK		 570static void reload_host_efer(struct vcpu_vmx *vmx)
571{
572 if (vmx->host_state.guest_efer_loaded) {
573 vmx->host_state.guest_efer_loaded = 0;
574 load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1);
575 }
576}
577
04d2cc77 578static void vmx_save_host_state(struct kvm_vcpu *vcpu)
33ed6329 579{
04d2cc77
AK		 580 struct vcpu_vmx *vmx = to_vmx(vcpu);
581
a2fa3e9f 582 if (vmx->host_state.loaded)
33ed6329
AK		 583 return;
584
a2fa3e9f 585 vmx->host_state.loaded = 1;
33ed6329
AK		 586 /*
587 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
588 * allow segment selectors with cpl > 0 or ti == 1.
589 */
d6e88aec 590 vmx->host_state.ldt_sel = kvm_read_ldt();
152d3f2f 591 vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel;
d6e88aec 592 vmx->host_state.fs_sel = kvm_read_fs();
152d3f2f 593 if (!(vmx->host_state.fs_sel & 7)) {
a2fa3e9f 594 vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel);
152d3f2f
LV		 595 vmx->host_state.fs_reload_needed = 0;
596 } else {
33ed6329 597 vmcs_write16(HOST_FS_SELECTOR, 0);
152d3f2f 598 vmx->host_state.fs_reload_needed = 1;
33ed6329 599 }
d6e88aec 600 vmx->host_state.gs_sel = kvm_read_gs();
a2fa3e9f
GH		 601 if (!(vmx->host_state.gs_sel & 7))
602 vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel);
33ed6329
AK		 603 else {
604 vmcs_write16(HOST_GS_SELECTOR, 0);
152d3f2f 605 vmx->host_state.gs_ldt_reload_needed = 1;
33ed6329
AK		 606 }
607
608#ifdef CONFIG_X86_64
609 vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
610 vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
611#else
a2fa3e9f
GH		 612 vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel));
613 vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel));
33ed6329 614#endif
707c0874
AK		 615
616#ifdef CONFIG_X86_64
d77c26fc 617 if (is_long_mode(&vmx->vcpu))
a2fa3e9f
GH		 618 save_msrs(vmx->host_msrs +
619 vmx->msr_offset_kernel_gs_base, 1);
d77c26fc 620
707c0874 621#endif
a2fa3e9f 622 load_msrs(vmx->guest_msrs, vmx->save_nmsrs);
51c6cf66 623 load_transition_efer(vmx);
33ed6329
AK		 624}
625
a9b21b62 626static void __vmx_load_host_state(struct vcpu_vmx *vmx)
33ed6329 627{
15ad7146 628 unsigned long flags;
33ed6329 629
a2fa3e9f 630 if (!vmx->host_state.loaded)
33ed6329
AK		 631 return;
632
e1beb1d3 633 ++vmx->vcpu.stat.host_state_reload;
a2fa3e9f 634 vmx->host_state.loaded = 0;
152d3f2f 635 if (vmx->host_state.fs_reload_needed)
d6e88aec 636 kvm_load_fs(vmx->host_state.fs_sel);
152d3f2f 637 if (vmx->host_state.gs_ldt_reload_needed) {
d6e88aec 638 kvm_load_ldt(vmx->host_state.ldt_sel);
33ed6329
AK		 639 /*
640 * If we have to reload gs, we must take care to
641 * preserve our gs base.
642 */
15ad7146 643 local_irq_save(flags);
d6e88aec 644 kvm_load_gs(vmx->host_state.gs_sel);
33ed6329
AK		 645#ifdef CONFIG_X86_64
646 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
647#endif
15ad7146 648 local_irq_restore(flags);
33ed6329 649 }
152d3f2f 650 reload_tss();
a2fa3e9f
GH		 651 save_msrs(vmx->guest_msrs, vmx->save_nmsrs);
652 load_msrs(vmx->host_msrs, vmx->save_nmsrs);
51c6cf66 653 reload_host_efer(vmx);
33ed6329
AK		 654}
655
a9b21b62
AK		 656static void vmx_load_host_state(struct vcpu_vmx *vmx)
657{
658 preempt_disable();
659 __vmx_load_host_state(vmx);
660 preempt_enable();
661}
662
6aa8b732
AK		 663/*
664 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
665 * vcpu mutex is already taken.
666 */
15ad7146 667static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
6aa8b732 668{
a2fa3e9f
GH		 669 struct vcpu_vmx *vmx = to_vmx(vcpu);
670 u64 phys_addr = __pa(vmx->vmcs);
019960ae 671 u64 tsc_this, delta, new_offset;
6aa8b732 672
a3d7f85f 673 if (vcpu->cpu != cpu) {
8b9cf98c 674 vcpu_clear(vmx);
2f599714 675 kvm_migrate_timers(vcpu);
2384d2b3 676 vpid_sync_vcpu_all(vmx);
543e4243
AK		 677 local_irq_disable();
678 list_add(&vmx->local_vcpus_link,
679 &per_cpu(vcpus_on_cpu, cpu));
680 local_irq_enable();
a3d7f85f 681 }
6aa8b732 682
a2fa3e9f 683 if (per_cpu(current_vmcs, cpu) != vmx->vmcs) {
6aa8b732
AK		 684 u8 error;
685
a2fa3e9f 686 per_cpu(current_vmcs, cpu) = vmx->vmcs;
4ecac3fd 687 asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) "; setna %0"
6aa8b732
AK		 688 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
689 : "cc");
690 if (error)
691 printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
a2fa3e9f 692 vmx->vmcs, phys_addr);
6aa8b732
AK		 693 }
694
695 if (vcpu->cpu != cpu) {
696 struct descriptor_table dt;
697 unsigned long sysenter_esp;
698
699 vcpu->cpu = cpu;
700 /*
701 * Linux uses per-cpu TSS and GDT, so set these when switching
702 * processors.
703 */
d6e88aec
AK		 704 vmcs_writel(HOST_TR_BASE, kvm_read_tr_base()); /* 22.2.4 */
705 kvm_get_gdt(&dt);
6aa8b732
AK		 706 vmcs_writel(HOST_GDTR_BASE, dt.base); /* 22.2.4 */
707
708 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
709 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
7700270e
AK		 710
711 /*
712 * Make sure the time stamp counter is monotonous.
713 */
714 rdtscll(tsc_this);
019960ae
AK		 715 if (tsc_this < vcpu->arch.host_tsc) {
716 delta = vcpu->arch.host_tsc - tsc_this;
717 new_offset = vmcs_read64(TSC_OFFSET) + delta;
718 vmcs_write64(TSC_OFFSET, new_offset);
719 }
6aa8b732 720 }
6aa8b732
AK		 721}
722
723static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
724{
a9b21b62 725 __vmx_load_host_state(to_vmx(vcpu));
6aa8b732
AK		 726}
727
5fd86fcf
AK		 728static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
729{
730 if (vcpu->fpu_active)
731 return;
732 vcpu->fpu_active = 1;
707d92fa 733 vmcs_clear_bits(GUEST_CR0, X86_CR0_TS);
ad312c7c 734 if (vcpu->arch.cr0 & X86_CR0_TS)
707d92fa 735 vmcs_set_bits(GUEST_CR0, X86_CR0_TS);
5fd86fcf
AK		 736 update_exception_bitmap(vcpu);
737}
738
739static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
740{
741 if (!vcpu->fpu_active)
742 return;
743 vcpu->fpu_active = 0;
707d92fa 744 vmcs_set_bits(GUEST_CR0, X86_CR0_TS);
5fd86fcf
AK		 745 update_exception_bitmap(vcpu);
746}
747
6aa8b732
AK		 748static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
749{
750 return vmcs_readl(GUEST_RFLAGS);
751}
752
753static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
754{
56b237e3 755 if (vcpu->arch.rmode.vm86_active)
053de044 756 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
6aa8b732
AK		 757 vmcs_writel(GUEST_RFLAGS, rflags);
758}
759
2809f5d2
GC		 760static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
761{
762 u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
763 int ret = 0;
764
765 if (interruptibility & GUEST_INTR_STATE_STI)
766 ret |= X86_SHADOW_INT_STI;
767 if (interruptibility & GUEST_INTR_STATE_MOV_SS)
768 ret |= X86_SHADOW_INT_MOV_SS;
769
770 return ret & mask;
771}
772
773static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
774{
775 u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
776 u32 interruptibility = interruptibility_old;
777
778 interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
779
780 if (mask & X86_SHADOW_INT_MOV_SS)
781 interruptibility |= GUEST_INTR_STATE_MOV_SS;
782 if (mask & X86_SHADOW_INT_STI)
783 interruptibility |= GUEST_INTR_STATE_STI;
784
785 if ((interruptibility != interruptibility_old))
786 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
787}
788
6aa8b732
AK		 789static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
790{
791 unsigned long rip;
6aa8b732 792
5fdbf976 793 rip = kvm_rip_read(vcpu);
6aa8b732 794 rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
5fdbf976 795 kvm_rip_write(vcpu, rip);
6aa8b732 796
2809f5d2
GC		 797 /* skipping an emulated instruction also counts */
798 vmx_set_interrupt_shadow(vcpu, 0);
6aa8b732
AK		 799}
800
298101da
AK		 801static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
802 bool has_error_code, u32 error_code)
803{
77ab6db0 804 struct vcpu_vmx *vmx = to_vmx(vcpu);
8ab2d2e2 805 u32 intr_info = nr | INTR_INFO_VALID_MASK;
77ab6db0 806
8ab2d2e2 807 if (has_error_code) {
77ab6db0 808 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
8ab2d2e2
JK		 809 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
810 }
77ab6db0 811
56b237e3 812 if (vcpu->arch.rmode.vm86_active) {
77ab6db0
JK		 813 vmx->rmode.irq.pending = true;
814 vmx->rmode.irq.vector = nr;
815 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
ae0bb3e0
GN		 816 if (kvm_exception_is_soft(nr))
817 vmx->rmode.irq.rip +=
818 vmx->vcpu.arch.event_exit_inst_len;
8ab2d2e2
JK		 819 intr_info |= INTR_TYPE_SOFT_INTR;
820 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
77ab6db0
JK		 821 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
822 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
823 return;
824 }
825
66fd3f7f
GN		 826 if (kvm_exception_is_soft(nr)) {
827 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
828 vmx->vcpu.arch.event_exit_inst_len);
8ab2d2e2
JK		 829 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
830 } else
831 intr_info |= INTR_TYPE_HARD_EXCEPTION;
832
833 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
298101da
AK		 834}
835
a75beee6
ED		 836/*
837 * Swap MSR entry in host/guest MSR entry array.
838 */
54e11fa1 839#ifdef CONFIG_X86_64
8b9cf98c 840static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
a75beee6 841{
a2fa3e9f
GH		 842 struct kvm_msr_entry tmp;
843
844 tmp = vmx->guest_msrs[to];
845 vmx->guest_msrs[to] = vmx->guest_msrs[from];
846 vmx->guest_msrs[from] = tmp;
847 tmp = vmx->host_msrs[to];
848 vmx->host_msrs[to] = vmx->host_msrs[from];
849 vmx->host_msrs[from] = tmp;
a75beee6 850}
54e11fa1 851#endif
a75beee6 852
e38aea3e
AK		 853/*
854 * Set up the vmcs to automatically save and restore system
855 * msrs. Don't touch the 64-bit msrs if the guest is in legacy
856 * mode, as fiddling with msrs is very expensive.
857 */
8b9cf98c 858static void setup_msrs(struct vcpu_vmx *vmx)
e38aea3e 859{
2cc51560 860 int save_nmsrs;
5897297b 861 unsigned long *msr_bitmap;
e38aea3e 862
33f9c505 863 vmx_load_host_state(vmx);
a75beee6
ED		 864 save_nmsrs = 0;
865#ifdef CONFIG_X86_64
8b9cf98c 866 if (is_long_mode(&vmx->vcpu)) {
2cc51560
ED		 867 int index;
868
8b9cf98c 869 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
a75beee6 870 if (index >= 0)
8b9cf98c
RR		 871 move_msr_up(vmx, index, save_nmsrs++);
872 index = __find_msr_index(vmx, MSR_LSTAR);
a75beee6 873 if (index >= 0)
8b9cf98c
RR		 874 move_msr_up(vmx, index, save_nmsrs++);
875 index = __find_msr_index(vmx, MSR_CSTAR);
a75beee6 876 if (index >= 0)
8b9cf98c
RR		 877 move_msr_up(vmx, index, save_nmsrs++);
878 index = __find_msr_index(vmx, MSR_KERNEL_GS_BASE);
a75beee6 879 if (index >= 0)
8b9cf98c 880 move_msr_up(vmx, index, save_nmsrs++);
a75beee6
ED		 881 /*
882 * MSR_K6_STAR is only needed on long mode guests, and only
883 * if efer.sce is enabled.
884 */
8b9cf98c 885 index = __find_msr_index(vmx, MSR_K6_STAR);
ad312c7c 886 if ((index >= 0) && (vmx->vcpu.arch.shadow_efer & EFER_SCE))
8b9cf98c 887 move_msr_up(vmx, index, save_nmsrs++);
a75beee6
ED		 888 }
889#endif
a2fa3e9f 890 vmx->save_nmsrs = save_nmsrs;
e38aea3e 891
4d56c8a7 892#ifdef CONFIG_X86_64
a2fa3e9f 893 vmx->msr_offset_kernel_gs_base =
8b9cf98c 894 __find_msr_index(vmx, MSR_KERNEL_GS_BASE);
4d56c8a7 895#endif
8b9cf98c 896 vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER);
5897297b
AK		 897
898 if (cpu_has_vmx_msr_bitmap()) {
899 if (is_long_mode(&vmx->vcpu))
900 msr_bitmap = vmx_msr_bitmap_longmode;
901 else
902 msr_bitmap = vmx_msr_bitmap_legacy;
903
904 vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
905 }
e38aea3e
AK		 906}
907
6aa8b732
AK		 908/*
909 * reads and returns guest's timestamp counter "register"
910 * guest_tsc = host_tsc + tsc_offset -- 21.3
911 */
912static u64 guest_read_tsc(void)
913{
914 u64 host_tsc, tsc_offset;
915
916 rdtscll(host_tsc);
917 tsc_offset = vmcs_read64(TSC_OFFSET);
918 return host_tsc + tsc_offset;
919}
920
921/*
922 * writes 'guest_tsc' into guest's timestamp counter "register"
923 * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
924 */
53f658b3 925static void guest_write_tsc(u64 guest_tsc, u64 host_tsc)
6aa8b732 926{
6aa8b732
AK		 927 vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
928}
929
6aa8b732
AK		 930/*
931 * Reads an msr value (of 'msr_index') into 'pdata'.
932 * Returns 0 on success, non-0 otherwise.
933 * Assumes vcpu_load() was already called.
934 */
935static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
936{
937 u64 data;
a2fa3e9f 938 struct kvm_msr_entry *msr;
6aa8b732
AK		 939
940 if (!pdata) {
941 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
942 return -EINVAL;
943 }
944
945 switch (msr_index) {
05b3e0c2 946#ifdef CONFIG_X86_64
6aa8b732
AK		 947 case MSR_FS_BASE:
948 data = vmcs_readl(GUEST_FS_BASE);
949 break;
950 case MSR_GS_BASE:
951 data = vmcs_readl(GUEST_GS_BASE);
952 break;
953 case MSR_EFER:
3bab1f5d 954 return kvm_get_msr_common(vcpu, msr_index, pdata);
6aa8b732 955#endif
af24a4e4 956 case MSR_IA32_TSC:
6aa8b732
AK		 957 data = guest_read_tsc();
958 break;
959 case MSR_IA32_SYSENTER_CS:
960 data = vmcs_read32(GUEST_SYSENTER_CS);
961 break;
962 case MSR_IA32_SYSENTER_EIP:
f5b42c33 963 data = vmcs_readl(GUEST_SYSENTER_EIP);
6aa8b732
AK		 964 break;
965 case MSR_IA32_SYSENTER_ESP:
f5b42c33 966 data = vmcs_readl(GUEST_SYSENTER_ESP);
6aa8b732 967 break;
6aa8b732 968 default:
516a1a7e 969 vmx_load_host_state(to_vmx(vcpu));
8b9cf98c 970 msr = find_msr_entry(to_vmx(vcpu), msr_index);
3bab1f5d
AK		 971 if (msr) {
972 data = msr->data;
973 break;
6aa8b732 974 }
3bab1f5d 975 return kvm_get_msr_common(vcpu, msr_index, pdata);
6aa8b732
AK		 976 }
977
978 *pdata = data;
979 return 0;
980}
981
982/*
983 * Writes msr value into into the appropriate "register".
984 * Returns 0 on success, non-0 otherwise.
985 * Assumes vcpu_load() was already called.
986 */
987static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
988{
a2fa3e9f
GH		 989 struct vcpu_vmx *vmx = to_vmx(vcpu);
990 struct kvm_msr_entry *msr;
53f658b3 991 u64 host_tsc;
2cc51560
ED		 992 int ret = 0;
993
6aa8b732 994 switch (msr_index) {
3bab1f5d 995 case MSR_EFER:
a9b21b62 996 vmx_load_host_state(vmx);
2cc51560 997 ret = kvm_set_msr_common(vcpu, msr_index, data);
2cc51560 998 break;
16175a79 999#ifdef CONFIG_X86_64
6aa8b732
AK		 1000 case MSR_FS_BASE:
1001 vmcs_writel(GUEST_FS_BASE, data);
1002 break;
1003 case MSR_GS_BASE:
1004 vmcs_writel(GUEST_GS_BASE, data);
1005 break;
1006#endif
1007 case MSR_IA32_SYSENTER_CS:
1008 vmcs_write32(GUEST_SYSENTER_CS, data);
1009 break;
1010 case MSR_IA32_SYSENTER_EIP:
f5b42c33 1011 vmcs_writel(GUEST_SYSENTER_EIP, data);
6aa8b732
AK		 1012 break;
1013 case MSR_IA32_SYSENTER_ESP:
f5b42c33 1014 vmcs_writel(GUEST_SYSENTER_ESP, data);
6aa8b732 1015 break;
af24a4e4 1016 case MSR_IA32_TSC:
53f658b3
MT		 1017 rdtscll(host_tsc);
1018 guest_write_tsc(data, host_tsc);
efa67e0d
CL		 1019 break;
1020 case MSR_P6_PERFCTR0:
1021 case MSR_P6_PERFCTR1:
1022 case MSR_P6_EVNTSEL0:
1023 case MSR_P6_EVNTSEL1:
1024 /*
1025 * Just discard all writes to the performance counters; this
1026 * should keep both older linux and windows 64-bit guests
1027 * happy
1028 */
1029 pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", msr_index, data);
1030
6aa8b732 1031 break;
468d472f
SY		 1032 case MSR_IA32_CR_PAT:
1033 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
1034 vmcs_write64(GUEST_IA32_PAT, data);
1035 vcpu->arch.pat = data;
1036 break;
1037 }
1038 /* Otherwise falls through to kvm_set_msr_common */
6aa8b732 1039 default:
a9b21b62 1040 vmx_load_host_state(vmx);
8b9cf98c 1041 msr = find_msr_entry(vmx, msr_index);
3bab1f5d
AK		 1042 if (msr) {
1043 msr->data = data;
1044 break;
6aa8b732 1045 }
2cc51560 1046 ret = kvm_set_msr_common(vcpu, msr_index, data);
6aa8b732
AK		 1047 }
1048
2cc51560 1049 return ret;
6aa8b732
AK		 1050}
1051
5fdbf976 1052static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
6aa8b732 1053{
5fdbf976
MT		 1054 __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
1055 switch (reg) {
1056 case VCPU_REGS_RSP:
1057 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
1058 break;
1059 case VCPU_REGS_RIP:
1060 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
1061 break;
6de4f3ad
AK		 1062 case VCPU_EXREG_PDPTR:
1063 if (enable_ept)
1064 ept_save_pdptrs(vcpu);
1065 break;
5fdbf976
MT		 1066 default:
1067 break;
1068 }
6aa8b732
AK		 1069}
1070
d0bfb940 1071static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
6aa8b732 1072{
d0bfb940
JK		 1073 int old_debug = vcpu->guest_debug;
1074 unsigned long flags;
6aa8b732 1075
d0bfb940
JK		 1076 vcpu->guest_debug = dbg->control;
1077 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
1078 vcpu->guest_debug = 0;
6aa8b732 1079
ae675ef0
JK		 1080 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1081 vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
1082 else
1083 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
1084
d0bfb940
JK		 1085 flags = vmcs_readl(GUEST_RFLAGS);
1086 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
1087 flags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
1088 else if (old_debug & KVM_GUESTDBG_SINGLESTEP)
6aa8b732 1089 flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
d0bfb940 1090 vmcs_writel(GUEST_RFLAGS, flags);
6aa8b732 1091
abd3f2d6 1092 update_exception_bitmap(vcpu);
6aa8b732
AK		 1093
1094 return 0;
1095}
1096
1097static __init int cpu_has_kvm_support(void)
1098{
6210e37b 1099 return cpu_has_vmx();
6aa8b732
AK		 1100}
1101
1102static __init int vmx_disabled_by_bios(void)
1103{
1104 u64 msr;
1105
1106 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
9ea542fa
SY		 1107 return (msr & (FEATURE_CONTROL_LOCKED |
1108 FEATURE_CONTROL_VMXON_ENABLED))
1109 == FEATURE_CONTROL_LOCKED;
62b3ffb8 1110 /* locked but not enabled */
6aa8b732
AK		 1111}
1112
774c47f1 1113static void hardware_enable(void *garbage)
6aa8b732
AK		 1114{
1115 int cpu = raw_smp_processor_id();
1116 u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
1117 u64 old;
1118
543e4243 1119 INIT_LIST_HEAD(&per_cpu(vcpus_on_cpu, cpu));
6aa8b732 1120 rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
9ea542fa
SY		 1121 if ((old & (FEATURE_CONTROL_LOCKED |
1122 FEATURE_CONTROL_VMXON_ENABLED))
1123 != (FEATURE_CONTROL_LOCKED |
1124 FEATURE_CONTROL_VMXON_ENABLED))
6aa8b732 1125 /* enable and lock */
62b3ffb8 1126 wrmsrl(MSR_IA32_FEATURE_CONTROL, old |
9ea542fa
SY		 1127 FEATURE_CONTROL_LOCKED |
1128 FEATURE_CONTROL_VMXON_ENABLED);
66aee91a 1129 write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
4ecac3fd
AK		 1130 asm volatile (ASM_VMX_VMXON_RAX
1131 : : "a"(&phys_addr), "m"(phys_addr)
6aa8b732
AK		 1132 : "memory", "cc");
1133}
1134
543e4243
AK		 1135static void vmclear_local_vcpus(void)
1136{
1137 int cpu = raw_smp_processor_id();
1138 struct vcpu_vmx *vmx, *n;
1139
1140 list_for_each_entry_safe(vmx, n, &per_cpu(vcpus_on_cpu, cpu),
1141 local_vcpus_link)
1142 __vcpu_clear(vmx);
1143}
1144
710ff4a8
EH		 1145
1146/* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
1147 * tricks.
1148 */
1149static void kvm_cpu_vmxoff(void)
6aa8b732 1150{
4ecac3fd 1151 asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
e693d71b 1152 write_cr4(read_cr4() & ~X86_CR4_VMXE);
6aa8b732
AK		 1153}
1154
710ff4a8
EH		 1155static void hardware_disable(void *garbage)
1156{
1157 vmclear_local_vcpus();
1158 kvm_cpu_vmxoff();
1159}
1160
1c3d14fe 1161static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
d77c26fc 1162 u32 msr, u32 *result)
1c3d14fe
YS		 1163{
1164 u32 vmx_msr_low, vmx_msr_high;
1165 u32 ctl = ctl_min | ctl_opt;
1166
1167 rdmsr(msr, vmx_msr_low, vmx_msr_high);
1168
1169 ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
1170 ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */
1171
1172 /* Ensure minimum (required) set of control bits are supported. */
1173 if (ctl_min & ~ctl)
002c7f7c 1174 return -EIO;
1c3d14fe
YS		 1175
1176 *result = ctl;
1177 return 0;
1178}
1179
002c7f7c 1180static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
6aa8b732
AK		 1181{
1182 u32 vmx_msr_low, vmx_msr_high;
d56f546d 1183 u32 min, opt, min2, opt2;
1c3d14fe
YS		 1184 u32 _pin_based_exec_control = 0;
1185 u32 _cpu_based_exec_control = 0;
f78e0e2e 1186 u32 _cpu_based_2nd_exec_control = 0;
1c3d14fe
YS		 1187 u32 _vmexit_control = 0;
1188 u32 _vmentry_control = 0;
1189
1190 min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
f08864b4 1191 opt = PIN_BASED_VIRTUAL_NMIS;
1c3d14fe
YS		 1192 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
1193 &_pin_based_exec_control) < 0)
002c7f7c 1194 return -EIO;
1c3d14fe
YS		 1195
1196 min = CPU_BASED_HLT_EXITING |
1197#ifdef CONFIG_X86_64
1198 CPU_BASED_CR8_LOAD_EXITING |
1199 CPU_BASED_CR8_STORE_EXITING |
1200#endif
d56f546d
SY		 1201 CPU_BASED_CR3_LOAD_EXITING |
1202 CPU_BASED_CR3_STORE_EXITING |
1c3d14fe
YS		 1203 CPU_BASED_USE_IO_BITMAPS |
1204 CPU_BASED_MOV_DR_EXITING |
a7052897
MT		 1205 CPU_BASED_USE_TSC_OFFSETING |
1206 CPU_BASED_INVLPG_EXITING;
f78e0e2e 1207 opt = CPU_BASED_TPR_SHADOW |
25c5f225 1208 CPU_BASED_USE_MSR_BITMAPS |
f78e0e2e 1209 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1c3d14fe
YS		 1210 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
1211 &_cpu_based_exec_control) < 0)
002c7f7c 1212 return -EIO;
6e5d865c
YS		 1213#ifdef CONFIG_X86_64
1214 if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
1215 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
1216 ~CPU_BASED_CR8_STORE_EXITING;
1217#endif
f78e0e2e 1218 if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
d56f546d
SY		 1219 min2 = 0;
1220 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2384d2b3 1221 SECONDARY_EXEC_WBINVD_EXITING |
d56f546d 1222 SECONDARY_EXEC_ENABLE_VPID |
3a624e29
NK		 1223 SECONDARY_EXEC_ENABLE_EPT |
1224 SECONDARY_EXEC_UNRESTRICTED_GUEST;
d56f546d
SY		 1225 if (adjust_vmx_controls(min2, opt2,
1226 MSR_IA32_VMX_PROCBASED_CTLS2,
f78e0e2e
SY		 1227 &_cpu_based_2nd_exec_control) < 0)
1228 return -EIO;
1229 }
1230#ifndef CONFIG_X86_64
1231 if (!(_cpu_based_2nd_exec_control &
1232 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
1233 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
1234#endif
d56f546d 1235 if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
a7052897
MT		 1236 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
1237 enabled */
d56f546d 1238 min &= ~(CPU_BASED_CR3_LOAD_EXITING |
a7052897
MT		 1239 CPU_BASED_CR3_STORE_EXITING |
1240 CPU_BASED_INVLPG_EXITING);
d56f546d
SY		 1241 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
1242 &_cpu_based_exec_control) < 0)
1243 return -EIO;
1244 rdmsr(MSR_IA32_VMX_EPT_VPID_CAP,
1245 vmx_capability.ept, vmx_capability.vpid);
1246 }
1c3d14fe
YS		 1247
1248 min = 0;
1249#ifdef CONFIG_X86_64
1250 min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
1251#endif
468d472f 1252 opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT;
1c3d14fe
YS		 1253 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
1254 &_vmexit_control) < 0)
002c7f7c 1255 return -EIO;
1c3d14fe 1256
468d472f
SY		 1257 min = 0;
1258 opt = VM_ENTRY_LOAD_IA32_PAT;
1c3d14fe
YS		 1259 if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
1260 &_vmentry_control) < 0)
002c7f7c 1261 return -EIO;
6aa8b732 1262
c68876fd 1263 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
1c3d14fe
YS		 1264
1265 /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
1266 if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
002c7f7c 1267 return -EIO;
1c3d14fe
YS		 1268
1269#ifdef CONFIG_X86_64
1270 /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
1271 if (vmx_msr_high & (1u<<16))
002c7f7c 1272 return -EIO;
1c3d14fe
YS		 1273#endif
1274
1275 /* Require Write-Back (WB) memory type for VMCS accesses. */
1276 if (((vmx_msr_high >> 18) & 15) != 6)
002c7f7c 1277 return -EIO;
1c3d14fe 1278
002c7f7c
YS		 1279 vmcs_conf->size = vmx_msr_high & 0x1fff;
1280 vmcs_conf->order = get_order(vmcs_config.size);
1281 vmcs_conf->revision_id = vmx_msr_low;
1c3d14fe 1282
002c7f7c
YS		 1283 vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
1284 vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
f78e0e2e 1285 vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
002c7f7c
YS		 1286 vmcs_conf->vmexit_ctrl = _vmexit_control;
1287 vmcs_conf->vmentry_ctrl = _vmentry_control;
1c3d14fe
YS		 1288
1289 return 0;
c68876fd 1290}
6aa8b732
AK		 1291
1292static struct vmcs *alloc_vmcs_cpu(int cpu)
1293{
1294 int node = cpu_to_node(cpu);
1295 struct page *pages;
1296 struct vmcs *vmcs;
1297
6484eb3e 1298 pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order);
6aa8b732
AK		 1299 if (!pages)
1300 return NULL;
1301 vmcs = page_address(pages);
1c3d14fe
YS		 1302 memset(vmcs, 0, vmcs_config.size);
1303 vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */
6aa8b732
AK		 1304 return vmcs;
1305}
1306
1307static struct vmcs *alloc_vmcs(void)
1308{
d3b2c338 1309 return alloc_vmcs_cpu(raw_smp_processor_id());
6aa8b732
AK		 1310}
1311
1312static void free_vmcs(struct vmcs *vmcs)
1313{
1c3d14fe 1314 free_pages((unsigned long)vmcs, vmcs_config.order);
6aa8b732
AK		 1315}
1316
39959588 1317static void free_kvm_area(void)
6aa8b732
AK		 1318{
1319 int cpu;
1320
1321 for_each_online_cpu(cpu)
1322 free_vmcs(per_cpu(vmxarea, cpu));
1323}
1324
6aa8b732
AK		 1325static __init int alloc_kvm_area(void)
1326{
1327 int cpu;
1328
1329 for_each_online_cpu(cpu) {
1330 struct vmcs *vmcs;
1331
1332 vmcs = alloc_vmcs_cpu(cpu);
1333 if (!vmcs) {
1334 free_kvm_area();
1335 return -ENOMEM;
1336 }
1337
1338 per_cpu(vmxarea, cpu) = vmcs;
1339 }
1340 return 0;
1341}
1342
1343static __init int hardware_setup(void)
1344{
002c7f7c
YS		 1345 if (setup_vmcs_config(&vmcs_config) < 0)
1346 return -EIO;
50a37eb4
JR		 1347
1348 if (boot_cpu_has(X86_FEATURE_NX))
1349 kvm_enable_efer_bits(EFER_NX);
1350
93ba03c2
SY		 1351 if (!cpu_has_vmx_vpid())
1352 enable_vpid = 0;
1353
3a624e29 1354 if (!cpu_has_vmx_ept()) {
93ba03c2 1355 enable_ept = 0;
3a624e29
NK		 1356 enable_unrestricted_guest = 0;
1357 }
1358
1359 if (!cpu_has_vmx_unrestricted_guest())
1360 enable_unrestricted_guest = 0;
93ba03c2
SY		 1361
1362 if (!cpu_has_vmx_flexpriority())
1363 flexpriority_enabled = 0;
1364
95ba8273
GN		 1365 if (!cpu_has_vmx_tpr_shadow())
1366 kvm_x86_ops->update_cr8_intercept = NULL;
1367
6aa8b732
AK		 1368 return alloc_kvm_area();
1369}
1370
1371static __exit void hardware_unsetup(void)
1372{
1373 free_kvm_area();
1374}
1375
6aa8b732
AK		 1376static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
1377{
1378 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1379
6af11b9e 1380 if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
6aa8b732
AK		 1381 vmcs_write16(sf->selector, save->selector);
1382 vmcs_writel(sf->base, save->base);
1383 vmcs_write32(sf->limit, save->limit);
1384 vmcs_write32(sf->ar_bytes, save->ar);
1385 } else {
1386 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
1387 << AR_DPL_SHIFT;
1388 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
1389 }
1390}
1391
1392static void enter_pmode(struct kvm_vcpu *vcpu)
1393{
1394 unsigned long flags;
a89a8fb9 1395 struct vcpu_vmx *vmx = to_vmx(vcpu);
6aa8b732 1396
a89a8fb9 1397 vmx->emulation_required = 1;
56b237e3 1398 vcpu->arch.rmode.vm86_active = 0;
6aa8b732 1399
ad312c7c
ZX		 1400 vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base);
1401 vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit);
1402 vmcs_write32(GUEST_TR_AR_BYTES, vcpu->arch.rmode.tr.ar);
6aa8b732
AK		 1403
1404 flags = vmcs_readl(GUEST_RFLAGS);
053de044 1405 flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM);
ad312c7c 1406 flags |= (vcpu->arch.rmode.save_iopl << IOPL_SHIFT);
6aa8b732
AK		 1407 vmcs_writel(GUEST_RFLAGS, flags);
1408
66aee91a
RR		 1409 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
1410 (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
6aa8b732
AK		 1411
1412 update_exception_bitmap(vcpu);
1413
a89a8fb9
MG		 1414 if (emulate_invalid_guest_state)
1415 return;
1416
ad312c7c
ZX		 1417 fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->arch.rmode.es);
1418 fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->arch.rmode.ds);
1419 fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->arch.rmode.gs);
1420 fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->arch.rmode.fs);
6aa8b732
AK		 1421
1422 vmcs_write16(GUEST_SS_SELECTOR, 0);
1423 vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
1424
1425 vmcs_write16(GUEST_CS_SELECTOR,
1426 vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
1427 vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1428}
1429
d77c26fc 1430static gva_t rmode_tss_base(struct kvm *kvm)
6aa8b732 1431{
bfc6d222 1432 if (!kvm->arch.tss_addr) {
cbc94022
IE		 1433 gfn_t base_gfn = kvm->memslots[0].base_gfn +
1434 kvm->memslots[0].npages - 3;
1435 return base_gfn << PAGE_SHIFT;
1436 }
bfc6d222 1437 return kvm->arch.tss_addr;
6aa8b732
AK		 1438}
1439
1440static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
1441{
1442 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1443
1444 save->selector = vmcs_read16(sf->selector);
1445 save->base = vmcs_readl(sf->base);
1446 save->limit = vmcs_read32(sf->limit);
1447 save->ar = vmcs_read32(sf->ar_bytes);
15b00f32
JK		 1448 vmcs_write16(sf->selector, save->base >> 4);
1449 vmcs_write32(sf->base, save->base & 0xfffff);
6aa8b732
AK		 1450 vmcs_write32(sf->limit, 0xffff);
1451 vmcs_write32(sf->ar_bytes, 0xf3);
1452}
1453
1454static void enter_rmode(struct kvm_vcpu *vcpu)
1455{
1456 unsigned long flags;
a89a8fb9 1457 struct vcpu_vmx *vmx = to_vmx(vcpu);
6aa8b732 1458
3a624e29
NK		 1459 if (enable_unrestricted_guest)
1460 return;
1461
a89a8fb9 1462 vmx->emulation_required = 1;
56b237e3 1463 vcpu->arch.rmode.vm86_active = 1;
6aa8b732 1464
ad312c7c 1465 vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
6aa8b732
AK		 1466 vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
1467
ad312c7c 1468 vcpu->arch.rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
6aa8b732
AK		 1469 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
1470
ad312c7c 1471 vcpu->arch.rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
6aa8b732
AK		 1472 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1473
1474 flags = vmcs_readl(GUEST_RFLAGS);
ad312c7c
ZX		 1475 vcpu->arch.rmode.save_iopl
1476 = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
6aa8b732 1477
053de044 1478 flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
6aa8b732
AK		 1479
1480 vmcs_writel(GUEST_RFLAGS, flags);
66aee91a 1481 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
6aa8b732
AK		 1482 update_exception_bitmap(vcpu);
1483
a89a8fb9
MG		 1484 if (emulate_invalid_guest_state)
1485 goto continue_rmode;
1486
6aa8b732
AK		 1487 vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
1488 vmcs_write32(GUEST_SS_LIMIT, 0xffff);
1489 vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
1490
1491 vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
abacf8df 1492 vmcs_write32(GUEST_CS_LIMIT, 0xffff);
8cb5b033
AK		 1493 if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
1494 vmcs_writel(GUEST_CS_BASE, 0xf0000);
6aa8b732
AK		 1495 vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
1496
ad312c7c
ZX		 1497 fix_rmode_seg(VCPU_SREG_ES, &vcpu->arch.rmode.es);
1498 fix_rmode_seg(VCPU_SREG_DS, &vcpu->arch.rmode.ds);
1499 fix_rmode_seg(VCPU_SREG_GS, &vcpu->arch.rmode.gs);
1500 fix_rmode_seg(VCPU_SREG_FS, &vcpu->arch.rmode.fs);
75880a01 1501
a89a8fb9 1502continue_rmode:
8668a3c4 1503 kvm_mmu_reset_context(vcpu);
b7ebfb05 1504 init_rmode(vcpu->kvm);
6aa8b732
AK		 1505}
1506
401d10de
AS		 1507static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
1508{
1509 struct vcpu_vmx *vmx = to_vmx(vcpu);
1510 struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
1511
1512 vcpu->arch.shadow_efer = efer;
1513 if (!msr)
1514 return;
1515 if (efer & EFER_LMA) {
1516 vmcs_write32(VM_ENTRY_CONTROLS,
1517 vmcs_read32(VM_ENTRY_CONTROLS) |
1518 VM_ENTRY_IA32E_MODE);
1519 msr->data = efer;
1520 } else {
1521 vmcs_write32(VM_ENTRY_CONTROLS,
1522 vmcs_read32(VM_ENTRY_CONTROLS) &
1523 ~VM_ENTRY_IA32E_MODE);
1524
1525 msr->data = efer & ~EFER_LME;
1526 }
1527 setup_msrs(vmx);
1528}
1529
05b3e0c2 1530#ifdef CONFIG_X86_64
6aa8b732
AK		 1531
1532static void enter_lmode(struct kvm_vcpu *vcpu)
1533{
1534 u32 guest_tr_ar;
1535
1536 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
1537 if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
1538 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
b8688d51 1539 __func__);
6aa8b732
AK		 1540 vmcs_write32(GUEST_TR_AR_BYTES,
1541 (guest_tr_ar & ~AR_TYPE_MASK)
1542 | AR_TYPE_BUSY_64_TSS);
1543 }
ad312c7c 1544 vcpu->arch.shadow_efer |= EFER_LMA;
401d10de 1545 vmx_set_efer(vcpu, vcpu->arch.shadow_efer);
6aa8b732
AK		 1546}
1547
1548static void exit_lmode(struct kvm_vcpu *vcpu)
1549{
ad312c7c 1550 vcpu->arch.shadow_efer &= ~EFER_LMA;
6aa8b732
AK		 1551
1552 vmcs_write32(VM_ENTRY_CONTROLS,
1553 vmcs_read32(VM_ENTRY_CONTROLS)
1e4e6e00 1554 & ~VM_ENTRY_IA32E_MODE);
6aa8b732
AK		 1555}
1556
1557#endif
1558
2384d2b3
SY		 1559static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
1560{
1561 vpid_sync_vcpu_all(to_vmx(vcpu));
089d034e 1562 if (enable_ept)
4e1096d2 1563 ept_sync_context(construct_eptp(vcpu->arch.mmu.root_hpa));
2384d2b3
SY		 1564}
1565
25c4c276 1566static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
399badf3 1567{
ad312c7c
ZX		 1568 vcpu->arch.cr4 &= KVM_GUEST_CR4_MASK;
1569 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK;
399badf3
AK		 1570}
1571
1439442c
SY		 1572static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
1573{
6de4f3ad
AK		 1574 if (!test_bit(VCPU_EXREG_PDPTR,
1575 (unsigned long *)&vcpu->arch.regs_dirty))
1576 return;
1577
1439442c 1578 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1439442c
SY		 1579 vmcs_write64(GUEST_PDPTR0, vcpu->arch.pdptrs[0]);
1580 vmcs_write64(GUEST_PDPTR1, vcpu->arch.pdptrs[1]);
1581 vmcs_write64(GUEST_PDPTR2, vcpu->arch.pdptrs[2]);
1582 vmcs_write64(GUEST_PDPTR3, vcpu->arch.pdptrs[3]);
1583 }
1584}
1585
8f5d549f
AK		 1586static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
1587{
1588 if (is_paging(vcpu) && is_pae(vcpu) && !is_long_mode(vcpu)) {
1589 vcpu->arch.pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
1590 vcpu->arch.pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
1591 vcpu->arch.pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
1592 vcpu->arch.pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
1593 }
6de4f3ad
AK		 1594
1595 __set_bit(VCPU_EXREG_PDPTR,
1596 (unsigned long *)&vcpu->arch.regs_avail);
1597 __set_bit(VCPU_EXREG_PDPTR,
1598 (unsigned long *)&vcpu->arch.regs_dirty);
8f5d549f
AK		 1599}
1600
1439442c
SY		 1601static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
1602
1603static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
1604 unsigned long cr0,
1605 struct kvm_vcpu *vcpu)
1606{
1607 if (!(cr0 & X86_CR0_PG)) {
1608 /* From paging/starting to nonpaging */
1609 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
65267ea1 1610 vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
1439442c
SY		 1611 (CPU_BASED_CR3_LOAD_EXITING |
1612 CPU_BASED_CR3_STORE_EXITING));
1613 vcpu->arch.cr0 = cr0;
1614 vmx_set_cr4(vcpu, vcpu->arch.cr4);
1439442c
SY		 1615 *hw_cr0 &= ~X86_CR0_WP;
1616 } else if (!is_paging(vcpu)) {
1617 /* From nonpaging to paging */
1618 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
65267ea1 1619 vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
1439442c
SY		 1620 ~(CPU_BASED_CR3_LOAD_EXITING |
1621 CPU_BASED_CR3_STORE_EXITING));
1622 vcpu->arch.cr0 = cr0;
1623 vmx_set_cr4(vcpu, vcpu->arch.cr4);
1624 if (!(vcpu->arch.cr0 & X86_CR0_WP))
1625 *hw_cr0 &= ~X86_CR0_WP;
1626 }
1627}
1628
1629static void ept_update_paging_mode_cr4(unsigned long *hw_cr4,
1630 struct kvm_vcpu *vcpu)
1631{
1632 if (!is_paging(vcpu)) {
1633 *hw_cr4 &= ~X86_CR4_PAE;
1634 *hw_cr4 |= X86_CR4_PSE;
1635 } else if (!(vcpu->arch.cr4 & X86_CR4_PAE))
1636 *hw_cr4 &= ~X86_CR4_PAE;
1637}
1638
6aa8b732
AK		 1639static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1640{
3a624e29
NK		 1641 unsigned long hw_cr0;
1642
1643 if (enable_unrestricted_guest)
1644 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK_UNRESTRICTED_GUEST)
1645 | KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
1646 else
1647 hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON;
1439442c 1648
5fd86fcf
AK		 1649 vmx_fpu_deactivate(vcpu);
1650
56b237e3 1651 if (vcpu->arch.rmode.vm86_active && (cr0 & X86_CR0_PE))
6aa8b732
AK		 1652 enter_pmode(vcpu);
1653
56b237e3 1654 if (!vcpu->arch.rmode.vm86_active && !(cr0 & X86_CR0_PE))
6aa8b732
AK		 1655 enter_rmode(vcpu);
1656
05b3e0c2 1657#ifdef CONFIG_X86_64
ad312c7c 1658 if (vcpu->arch.shadow_efer & EFER_LME) {
707d92fa 1659 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
6aa8b732 1660 enter_lmode(vcpu);
707d92fa 1661 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
6aa8b732
AK		 1662 exit_lmode(vcpu);
1663 }
1664#endif
1665
089d034e 1666 if (enable_ept)
1439442c
SY		 1667 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
1668
6aa8b732 1669 vmcs_writel(CR0_READ_SHADOW, cr0);
1439442c 1670 vmcs_writel(GUEST_CR0, hw_cr0);
ad312c7c 1671 vcpu->arch.cr0 = cr0;
5fd86fcf 1672
707d92fa 1673 if (!(cr0 & X86_CR0_TS) || !(cr0 & X86_CR0_PE))
5fd86fcf 1674 vmx_fpu_activate(vcpu);
6aa8b732
AK		 1675}
1676
1439442c
SY		 1677static u64 construct_eptp(unsigned long root_hpa)
1678{
1679 u64 eptp;
1680
1681 /* TODO write the value reading from MSR */
1682 eptp = VMX_EPT_DEFAULT_MT |
1683 VMX_EPT_DEFAULT_GAW << VMX_EPT_GAW_EPTP_SHIFT;
1684 eptp |= (root_hpa & PAGE_MASK);
1685
1686 return eptp;
1687}
1688
6aa8b732
AK		 1689static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1690{
1439442c
SY		 1691 unsigned long guest_cr3;
1692 u64 eptp;
1693
1694 guest_cr3 = cr3;
089d034e 1695 if (enable_ept) {
1439442c
SY		 1696 eptp = construct_eptp(cr3);
1697 vmcs_write64(EPT_POINTER, eptp);
1439442c
SY		 1698 guest_cr3 = is_paging(vcpu) ? vcpu->arch.cr3 :
1699 VMX_EPT_IDENTITY_PAGETABLE_ADDR;
1700 }
1701
2384d2b3 1702 vmx_flush_tlb(vcpu);
1439442c 1703 vmcs_writel(GUEST_CR3, guest_cr3);
ad312c7c 1704 if (vcpu->arch.cr0 & X86_CR0_PE)
5fd86fcf 1705 vmx_fpu_deactivate(vcpu);
6aa8b732
AK		 1706}
1707
1708static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1709{
56b237e3 1710 unsigned long hw_cr4 = cr4 | (vcpu->arch.rmode.vm86_active ?
1439442c
SY		 1711 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
1712
ad312c7c 1713 vcpu->arch.cr4 = cr4;
089d034e 1714 if (enable_ept)
1439442c
SY		 1715 ept_update_paging_mode_cr4(&hw_cr4, vcpu);
1716
1717 vmcs_writel(CR4_READ_SHADOW, cr4);
1718 vmcs_writel(GUEST_CR4, hw_cr4);
6aa8b732
AK		 1719}
1720
6aa8b732
AK		 1721static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1722{
1723 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1724
1725 return vmcs_readl(sf->base);
1726}
1727
1728static void vmx_get_segment(struct kvm_vcpu *vcpu,
1729 struct kvm_segment *var, int seg)
1730{
1731 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1732 u32 ar;
1733
1734 var->base = vmcs_readl(sf->base);
1735 var->limit = vmcs_read32(sf->limit);
1736 var->selector = vmcs_read16(sf->selector);
1737 ar = vmcs_read32(sf->ar_bytes);
9fd4a3b7 1738 if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
6aa8b732
AK		 1739 ar = 0;
1740 var->type = ar & 15;
1741 var->s = (ar >> 4) & 1;
1742 var->dpl = (ar >> 5) & 3;
1743 var->present = (ar >> 7) & 1;
1744 var->avl = (ar >> 12) & 1;
1745 var->l = (ar >> 13) & 1;
1746 var->db = (ar >> 14) & 1;
1747 var->g = (ar >> 15) & 1;
1748 var->unusable = (ar >> 16) & 1;
1749}
1750
2e4d2653
IE		 1751static int vmx_get_cpl(struct kvm_vcpu *vcpu)
1752{
1753 struct kvm_segment kvm_seg;
1754
1755 if (!(vcpu->arch.cr0 & X86_CR0_PE)) /* if real mode */
1756 return 0;
1757
1758 if (vmx_get_rflags(vcpu) & X86_EFLAGS_VM) /* if virtual 8086 */
1759 return 3;
1760
1761 vmx_get_segment(vcpu, &kvm_seg, VCPU_SREG_CS);
1762 return kvm_seg.selector & 3;
1763}
1764
653e3108 1765static u32 vmx_segment_access_rights(struct kvm_segment *var)
6aa8b732 1766{
6aa8b732
AK		 1767 u32 ar;
1768
653e3108 1769 if (var->unusable)
6aa8b732
AK		 1770 ar = 1 << 16;
1771 else {
1772 ar = var->type & 15;
1773 ar |= (var->s & 1) << 4;
1774 ar |= (var->dpl & 3) << 5;
1775 ar |= (var->present & 1) << 7;
1776 ar |= (var->avl & 1) << 12;
1777 ar |= (var->l & 1) << 13;
1778 ar |= (var->db & 1) << 14;
1779 ar |= (var->g & 1) << 15;
1780 }
f7fbf1fd
UL		 1781 if (ar == 0) /* a 0 value means unusable */
1782 ar = AR_UNUSABLE_MASK;
653e3108
AK		 1783
1784 return ar;
1785}
1786
1787static void vmx_set_segment(struct kvm_vcpu *vcpu,
1788 struct kvm_segment *var, int seg)
1789{
1790 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1791 u32 ar;
1792
56b237e3 1793 if (vcpu->arch.rmode.vm86_active && seg == VCPU_SREG_TR) {
ad312c7c
ZX		 1794 vcpu->arch.rmode.tr.selector = var->selector;
1795 vcpu->arch.rmode.tr.base = var->base;
1796 vcpu->arch.rmode.tr.limit = var->limit;
1797 vcpu->arch.rmode.tr.ar = vmx_segment_access_rights(var);
653e3108
AK		 1798 return;
1799 }
1800 vmcs_writel(sf->base, var->base);
1801 vmcs_write32(sf->limit, var->limit);
1802 vmcs_write16(sf->selector, var->selector);
56b237e3 1803 if (vcpu->arch.rmode.vm86_active && var->s) {
653e3108
AK		 1804 /*
1805 * Hack real-mode segments into vm86 compatibility.
1806 */
1807 if (var->base == 0xffff0000 && var->selector == 0xf000)
1808 vmcs_writel(sf->base, 0xf0000);
1809 ar = 0xf3;
1810 } else
1811 ar = vmx_segment_access_rights(var);
3a624e29
NK		 1812
1813 /*
1814 * Fix the "Accessed" bit in AR field of segment registers for older
1815 * qemu binaries.
1816 * IA32 arch specifies that at the time of processor reset the
1817 * "Accessed" bit in the AR field of segment registers is 1. And qemu
1818 * is setting it to 0 in the usedland code. This causes invalid guest
1819 * state vmexit when "unrestricted guest" mode is turned on.
1820 * Fix for this setup issue in cpu_reset is being pushed in the qemu
1821 * tree. Newer qemu binaries with that qemu fix would not need this
1822 * kvm hack.
1823 */
1824 if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
1825 ar |= 0x1; /* Accessed */
1826
6aa8b732
AK		 1827 vmcs_write32(sf->ar_bytes, ar);
1828}
1829
6aa8b732
AK		 1830static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1831{
1832 u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
1833
1834 *db = (ar >> 14) & 1;
1835 *l = (ar >> 13) & 1;
1836}
1837
1838static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1839{
1840 dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
1841 dt->base = vmcs_readl(GUEST_IDTR_BASE);
1842}
1843
1844static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1845{
1846 vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
1847 vmcs_writel(GUEST_IDTR_BASE, dt->base);
1848}
1849
1850static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1851{
1852 dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
1853 dt->base = vmcs_readl(GUEST_GDTR_BASE);
1854}
1855
1856static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1857{
1858 vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
1859 vmcs_writel(GUEST_GDTR_BASE, dt->base);
1860}
1861
648dfaa7
MG		 1862static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
1863{
1864 struct kvm_segment var;
1865 u32 ar;
1866
1867 vmx_get_segment(vcpu, &var, seg);
1868 ar = vmx_segment_access_rights(&var);
1869
1870 if (var.base != (var.selector << 4))
1871 return false;
1872 if (var.limit != 0xffff)
1873 return false;
1874 if (ar != 0xf3)
1875 return false;
1876
1877 return true;
1878}
1879
1880static bool code_segment_valid(struct kvm_vcpu *vcpu)
1881{
1882 struct kvm_segment cs;
1883 unsigned int cs_rpl;
1884
1885 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
1886 cs_rpl = cs.selector & SELECTOR_RPL_MASK;
1887
1872a3f4
AK		 1888 if (cs.unusable)
1889 return false;
648dfaa7
MG		 1890 if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK))
1891 return false;
1892 if (!cs.s)
1893 return false;
1872a3f4 1894 if (cs.type & AR_TYPE_WRITEABLE_MASK) {
648dfaa7
MG		 1895 if (cs.dpl > cs_rpl)
1896 return false;
1872a3f4 1897 } else {
648dfaa7
MG		 1898 if (cs.dpl != cs_rpl)
1899 return false;
1900 }
1901 if (!cs.present)
1902 return false;
1903
1904 /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
1905 return true;
1906}
1907
1908static bool stack_segment_valid(struct kvm_vcpu *vcpu)
1909{
1910 struct kvm_segment ss;
1911 unsigned int ss_rpl;
1912
1913 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
1914 ss_rpl = ss.selector & SELECTOR_RPL_MASK;
1915
1872a3f4
AK		 1916 if (ss.unusable)
1917 return true;
1918 if (ss.type != 3 && ss.type != 7)
648dfaa7
MG		 1919 return false;
1920 if (!ss.s)
1921 return false;
1922 if (ss.dpl != ss_rpl) /* DPL != RPL */
1923 return false;
1924 if (!ss.present)
1925 return false;
1926
1927 return true;
1928}
1929
1930static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
1931{
1932 struct kvm_segment var;
1933 unsigned int rpl;
1934
1935 vmx_get_segment(vcpu, &var, seg);
1936 rpl = var.selector & SELECTOR_RPL_MASK;
1937
1872a3f4
AK		 1938 if (var.unusable)
1939 return true;
648dfaa7
MG		 1940 if (!var.s)
1941 return false;
1942 if (!var.present)
1943 return false;
1944 if (~var.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK)) {
1945 if (var.dpl < rpl) /* DPL < RPL */
1946 return false;
1947 }
1948
1949 /* TODO: Add other members to kvm_segment_field to allow checking for other access
1950 * rights flags
1951 */
1952 return true;
1953}
1954
1955static bool tr_valid(struct kvm_vcpu *vcpu)
1956{
1957 struct kvm_segment tr;
1958
1959 vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
1960
1872a3f4
AK		 1961 if (tr.unusable)
1962 return false;
648dfaa7
MG		 1963 if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */
1964 return false;
1872a3f4 1965 if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
648dfaa7
MG		 1966 return false;
1967 if (!tr.present)
1968 return false;
1969
1970 return true;
1971}
1972
1973static bool ldtr_valid(struct kvm_vcpu *vcpu)
1974{
1975 struct kvm_segment ldtr;
1976
1977 vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
1978
1872a3f4
AK		 1979 if (ldtr.unusable)
1980 return true;
648dfaa7
MG		 1981 if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */
1982 return false;
1983 if (ldtr.type != 2)
1984 return false;
1985 if (!ldtr.present)
1986 return false;
1987
1988 return true;
1989}
1990
1991static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
1992{
1993 struct kvm_segment cs, ss;
1994
1995 vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
1996 vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
1997
1998 return ((cs.selector & SELECTOR_RPL_MASK) ==
1999 (ss.selector & SELECTOR_RPL_MASK));
2000}
2001
2002/*
2003 * Check if guest state is valid. Returns true if valid, false if
2004 * not.
2005 * We assume that registers are always usable
2006 */
2007static bool guest_state_valid(struct kvm_vcpu *vcpu)
2008{
2009 /* real mode guest state checks */
2010 if (!(vcpu->arch.cr0 & X86_CR0_PE)) {
2011 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
2012 return false;
2013 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
2014 return false;
2015 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
2016 return false;
2017 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
2018 return false;
2019 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
2020 return false;
2021 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
2022 return false;
2023 } else {
2024 /* protected mode guest state checks */
2025 if (!cs_ss_rpl_check(vcpu))
2026 return false;
2027 if (!code_segment_valid(vcpu))
2028 return false;
2029 if (!stack_segment_valid(vcpu))
2030 return false;
2031 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
2032 return false;
2033 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
2034 return false;
2035 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
2036 return false;
2037 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
2038 return false;
2039 if (!tr_valid(vcpu))
2040 return false;
2041 if (!ldtr_valid(vcpu))
2042 return false;
2043 }
2044 /* TODO:
2045 * - Add checks on RIP
2046 * - Add checks on RFLAGS
2047 */
2048
2049 return true;
2050}
2051
d77c26fc 2052static int init_rmode_tss(struct kvm *kvm)
6aa8b732 2053{
6aa8b732 2054 gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
195aefde 2055 u16 data = 0;
10589a46 2056 int ret = 0;
195aefde 2057 int r;
6aa8b732 2058
195aefde
IE		 2059 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2060 if (r < 0)
10589a46 2061 goto out;
195aefde 2062 data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
464d17c8
SY		 2063 r = kvm_write_guest_page(kvm, fn++, &data,
2064 TSS_IOPB_BASE_OFFSET, sizeof(u16));
195aefde 2065 if (r < 0)
10589a46 2066 goto out;
195aefde
IE		 2067 r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
2068 if (r < 0)
10589a46 2069 goto out;
195aefde
IE		 2070 r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
2071 if (r < 0)
10589a46 2072 goto out;
195aefde 2073 data = ~0;
10589a46
MT		 2074 r = kvm_write_guest_page(kvm, fn, &data,
2075 RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
2076 sizeof(u8));
195aefde 2077 if (r < 0)
10589a46
MT		 2078 goto out;
2079
2080 ret = 1;
2081out:
10589a46 2082 return ret;
6aa8b732
AK		 2083}
2084
b7ebfb05
SY		 2085static int init_rmode_identity_map(struct kvm *kvm)
2086{
2087 int i, r, ret;
2088 pfn_t identity_map_pfn;
2089 u32 tmp;
2090
089d034e 2091 if (!enable_ept)
b7ebfb05
SY		 2092 return 1;
2093 if (unlikely(!kvm->arch.ept_identity_pagetable)) {
2094 printk(KERN_ERR "EPT: identity-mapping pagetable "
2095 "haven't been allocated!\n");
2096 return 0;
2097 }
2098 if (likely(kvm->arch.ept_identity_pagetable_done))
2099 return 1;
2100 ret = 0;
2101 identity_map_pfn = VMX_EPT_IDENTITY_PAGETABLE_ADDR >> PAGE_SHIFT;
2102 r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
2103 if (r < 0)
2104 goto out;
2105 /* Set up identity-mapping pagetable for EPT in real mode */
2106 for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
2107 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
2108 _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
2109 r = kvm_write_guest_page(kvm, identity_map_pfn,
2110 &tmp, i * sizeof(tmp), sizeof(tmp));
2111 if (r < 0)
2112 goto out;
2113 }
2114 kvm->arch.ept_identity_pagetable_done = true;
2115 ret = 1;
2116out:
2117 return ret;
2118}
2119
6aa8b732
AK		 2120static void seg_setup(int seg)
2121{
2122 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3a624e29 2123 unsigned int ar;
6aa8b732
AK		 2124
2125 vmcs_write16(sf->selector, 0);
2126 vmcs_writel(sf->base, 0);
2127 vmcs_write32(sf->limit, 0xffff);
3a624e29
NK		 2128 if (enable_unrestricted_guest) {
2129 ar = 0x93;
2130 if (seg == VCPU_SREG_CS)
2131 ar |= 0x08; /* code segment */
2132 } else
2133 ar = 0xf3;
2134
2135 vmcs_write32(sf->ar_bytes, ar);
6aa8b732
AK		 2136}
2137
f78e0e2e
SY		 2138static int alloc_apic_access_page(struct kvm *kvm)
2139{
2140 struct kvm_userspace_memory_region kvm_userspace_mem;
2141 int r = 0;
2142
72dc67a6 2143 down_write(&kvm->slots_lock);
bfc6d222 2144 if (kvm->arch.apic_access_page)
f78e0e2e
SY		 2145 goto out;
2146 kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
2147 kvm_userspace_mem.flags = 0;
2148 kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL;
2149 kvm_userspace_mem.memory_size = PAGE_SIZE;
2150 r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2151 if (r)
2152 goto out;
72dc67a6 2153
bfc6d222 2154 kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00);
f78e0e2e 2155out:
72dc67a6 2156 up_write(&kvm->slots_lock);
f78e0e2e
SY		 2157 return r;
2158}
2159
b7ebfb05
SY		 2160static int alloc_identity_pagetable(struct kvm *kvm)
2161{
2162 struct kvm_userspace_memory_region kvm_userspace_mem;
2163 int r = 0;
2164
2165 down_write(&kvm->slots_lock);
2166 if (kvm->arch.ept_identity_pagetable)
2167 goto out;
2168 kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
2169 kvm_userspace_mem.flags = 0;
2170 kvm_userspace_mem.guest_phys_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
2171 kvm_userspace_mem.memory_size = PAGE_SIZE;
2172 r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0);
2173 if (r)
2174 goto out;
2175
b7ebfb05
SY		 2176 kvm->arch.ept_identity_pagetable = gfn_to_page(kvm,
2177 VMX_EPT_IDENTITY_PAGETABLE_ADDR >> PAGE_SHIFT);
b7ebfb05
SY		 2178out:
2179 up_write(&kvm->slots_lock);
2180 return r;
2181}
2182
2384d2b3
SY		 2183static void allocate_vpid(struct vcpu_vmx *vmx)
2184{
2185 int vpid;
2186
2187 vmx->vpid = 0;
919818ab 2188 if (!enable_vpid)
2384d2b3
SY		 2189 return;
2190 spin_lock(&vmx_vpid_lock);
2191 vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
2192 if (vpid < VMX_NR_VPIDS) {
2193 vmx->vpid = vpid;
2194 __set_bit(vpid, vmx_vpid_bitmap);
2195 }
2196 spin_unlock(&vmx_vpid_lock);
2197}
2198
5897297b 2199static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, u32 msr)
25c5f225 2200{
3e7c73e9 2201 int f = sizeof(unsigned long);
25c5f225
SY		 2202
2203 if (!cpu_has_vmx_msr_bitmap())
2204 return;
2205
2206 /*
2207 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
2208 * have the write-low and read-high bitmap offsets the wrong way round.
2209 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
2210 */
25c5f225 2211 if (msr <= 0x1fff) {
3e7c73e9
AK		 2212 __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
2213 __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
25c5f225
SY		 2214 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2215 msr &= 0x1fff;
3e7c73e9
AK		 2216 __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
2217 __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
25c5f225 2218 }
25c5f225
SY		 2219}
2220
5897297b
AK		 2221static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
2222{
2223 if (!longmode_only)
2224 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy, msr);
2225 __vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode, msr);
2226}
2227
6aa8b732
AK		 2228/*
2229 * Sets up the vmcs for emulated real mode.
2230 */
8b9cf98c 2231static int vmx_vcpu_setup(struct vcpu_vmx *vmx)
6aa8b732 2232{
468d472f 2233 u32 host_sysenter_cs, msr_low, msr_high;
6aa8b732 2234 u32 junk;
53f658b3 2235 u64 host_pat, tsc_this, tsc_base;
6aa8b732
AK		 2236 unsigned long a;
2237 struct descriptor_table dt;
2238 int i;
cd2276a7 2239 unsigned long kvm_vmx_return;
6e5d865c 2240 u32 exec_control;
6aa8b732 2241
6aa8b732 2242 /* I/O */
3e7c73e9
AK		 2243 vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
2244 vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
6aa8b732 2245
25c5f225 2246 if (cpu_has_vmx_msr_bitmap())
5897297b 2247 vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
25c5f225 2248
6aa8b732
AK		 2249 vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
2250
6aa8b732 2251 /* Control */
1c3d14fe
YS		 2252 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
2253 vmcs_config.pin_based_exec_ctrl);
6e5d865c
YS		 2254
2255 exec_control = vmcs_config.cpu_based_exec_ctrl;
2256 if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) {
2257 exec_control &= ~CPU_BASED_TPR_SHADOW;
2258#ifdef CONFIG_X86_64
2259 exec_control |= CPU_BASED_CR8_STORE_EXITING |
2260 CPU_BASED_CR8_LOAD_EXITING;
2261#endif
2262 }
089d034e 2263 if (!enable_ept)
d56f546d 2264 exec_control |= CPU_BASED_CR3_STORE_EXITING |
83dbc83a
MT		 2265 CPU_BASED_CR3_LOAD_EXITING |
2266 CPU_BASED_INVLPG_EXITING;
6e5d865c 2267 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
6aa8b732 2268
83ff3b9d
SY		 2269 if (cpu_has_secondary_exec_ctrls()) {
2270 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
2271 if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2272 exec_control &=
2273 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
2384d2b3
SY		 2274 if (vmx->vpid == 0)
2275 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
089d034e 2276 if (!enable_ept)
d56f546d 2277 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
3a624e29
NK		 2278 if (!enable_unrestricted_guest)
2279 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
83ff3b9d
SY		 2280 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
2281 }
f78e0e2e 2282
c7addb90
AK		 2283 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf);
2284 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf);
6aa8b732
AK		 2285 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
2286
2287 vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */
2288 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
2289 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
2290
2291 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
2292 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
2293 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
d6e88aec
AK		 2294 vmcs_write16(HOST_FS_SELECTOR, kvm_read_fs()); /* 22.2.4 */
2295 vmcs_write16(HOST_GS_SELECTOR, kvm_read_gs()); /* 22.2.4 */
6aa8b732 2296 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
05b3e0c2 2297#ifdef CONFIG_X86_64
6aa8b732
AK		 2298 rdmsrl(MSR_FS_BASE, a);
2299 vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
2300 rdmsrl(MSR_GS_BASE, a);
2301 vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
2302#else
2303 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
2304 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
2305#endif
2306
2307 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
2308
d6e88aec 2309 kvm_get_idt(&dt);
6aa8b732
AK		 2310 vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */
2311
d77c26fc 2312 asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
cd2276a7 2313 vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
2cc51560
ED		 2314 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
2315 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2316 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
6aa8b732
AK		 2317
2318 rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
2319 vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
2320 rdmsrl(MSR_IA32_SYSENTER_ESP, a);
2321 vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */
2322 rdmsrl(MSR_IA32_SYSENTER_EIP, a);
2323 vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */
2324
468d472f
SY		 2325 if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
2326 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2327 host_pat = msr_low | ((u64) msr_high << 32);
2328 vmcs_write64(HOST_IA32_PAT, host_pat);
2329 }
2330 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2331 rdmsr(MSR_IA32_CR_PAT, msr_low, msr_high);
2332 host_pat = msr_low | ((u64) msr_high << 32);
2333 /* Write the default value follow host pat */
2334 vmcs_write64(GUEST_IA32_PAT, host_pat);
2335 /* Keep arch.pat sync with GUEST_IA32_PAT */
2336 vmx->vcpu.arch.pat = host_pat;
2337 }
2338
6aa8b732
AK		 2339 for (i = 0; i < NR_VMX_MSR; ++i) {
2340 u32 index = vmx_msr_index[i];
2341 u32 data_low, data_high;
2342 u64 data;
a2fa3e9f 2343 int j = vmx->nmsrs;
6aa8b732
AK		 2344
2345 if (rdmsr_safe(index, &data_low, &data_high) < 0)
2346 continue;
432bd6cb
AK		 2347 if (wrmsr_safe(index, data_low, data_high) < 0)
2348 continue;
6aa8b732 2349 data = data_low | ((u64)data_high << 32);
a2fa3e9f
GH		 2350 vmx->host_msrs[j].index = index;
2351 vmx->host_msrs[j].reserved = 0;
2352 vmx->host_msrs[j].data = data;
2353 vmx->guest_msrs[j] = vmx->host_msrs[j];
2354 ++vmx->nmsrs;
6aa8b732 2355 }
6aa8b732 2356
1c3d14fe 2357 vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
6aa8b732
AK		 2358
2359 /* 22.2.1, 20.8.1 */
1c3d14fe
YS		 2360 vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
2361
e00c8cf2
AK		 2362 vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
2363 vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK);
2364
53f658b3
MT		 2365 tsc_base = vmx->vcpu.kvm->arch.vm_init_tsc;
2366 rdtscll(tsc_this);
2367 if (tsc_this < vmx->vcpu.kvm->arch.vm_init_tsc)
2368 tsc_base = tsc_this;
2369
2370 guest_write_tsc(0, tsc_base);
f78e0e2e 2371
e00c8cf2
AK		 2372 return 0;
2373}
2374
b7ebfb05
SY		 2375static int init_rmode(struct kvm *kvm)
2376{
2377 if (!init_rmode_tss(kvm))
2378 return 0;
2379 if (!init_rmode_identity_map(kvm))
2380 return 0;
2381 return 1;
2382}
2383
e00c8cf2
AK		 2384static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
2385{
2386 struct vcpu_vmx *vmx = to_vmx(vcpu);
2387 u64 msr;
2388 int ret;
2389
5fdbf976 2390 vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP));
3200f405 2391 down_read(&vcpu->kvm->slots_lock);
b7ebfb05 2392 if (!init_rmode(vmx->vcpu.kvm)) {
e00c8cf2
AK		 2393 ret = -ENOMEM;
2394 goto out;
2395 }
2396
56b237e3 2397 vmx->vcpu.arch.rmode.vm86_active = 0;
e00c8cf2 2398
3b86cd99
JK		 2399 vmx->soft_vnmi_blocked = 0;
2400
ad312c7c 2401 vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
2d3ad1f4 2402 kvm_set_cr8(&vmx->vcpu, 0);
e00c8cf2
AK		 2403 msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
2404 if (vmx->vcpu.vcpu_id == 0)
2405 msr |= MSR_IA32_APICBASE_BSP;
2406 kvm_set_apic_base(&vmx->vcpu, msr);
2407
2408 fx_init(&vmx->vcpu);
2409
5706be0d 2410 seg_setup(VCPU_SREG_CS);
e00c8cf2
AK		 2411 /*
2412 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
2413 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh.
2414 */
2415 if (vmx->vcpu.vcpu_id == 0) {
2416 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
2417 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
2418 } else {
ad312c7c
ZX		 2419 vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8);
2420 vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12);
e00c8cf2 2421 }
e00c8cf2
AK		 2422
2423 seg_setup(VCPU_SREG_DS);
2424 seg_setup(VCPU_SREG_ES);
2425 seg_setup(VCPU_SREG_FS);
2426 seg_setup(VCPU_SREG_GS);
2427 seg_setup(VCPU_SREG_SS);
2428
2429 vmcs_write16(GUEST_TR_SELECTOR, 0);
2430 vmcs_writel(GUEST_TR_BASE, 0);
2431 vmcs_write32(GUEST_TR_LIMIT, 0xffff);
2432 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2433
2434 vmcs_write16(GUEST_LDTR_SELECTOR, 0);
2435 vmcs_writel(GUEST_LDTR_BASE, 0);
2436 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
2437 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
2438
2439 vmcs_write32(GUEST_SYSENTER_CS, 0);
2440 vmcs_writel(GUEST_SYSENTER_ESP, 0);
2441 vmcs_writel(GUEST_SYSENTER_EIP, 0);
2442
2443 vmcs_writel(GUEST_RFLAGS, 0x02);
2444 if (vmx->vcpu.vcpu_id == 0)
5fdbf976 2445 kvm_rip_write(vcpu, 0xfff0);
e00c8cf2 2446 else
5fdbf976
MT		 2447 kvm_rip_write(vcpu, 0);
2448 kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
e00c8cf2 2449
e00c8cf2
AK		 2450 vmcs_writel(GUEST_DR7, 0x400);
2451
2452 vmcs_writel(GUEST_GDTR_BASE, 0);
2453 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
2454
2455 vmcs_writel(GUEST_IDTR_BASE, 0);
2456 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
2457
2458 vmcs_write32(GUEST_ACTIVITY_STATE, 0);
2459 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
2460 vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
2461
e00c8cf2
AK		 2462 /* Special registers */
2463 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
2464
2465 setup_msrs(vmx);
2466
6aa8b732
AK		 2467 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
2468
f78e0e2e
SY		 2469 if (cpu_has_vmx_tpr_shadow()) {
2470 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
2471 if (vm_need_tpr_shadow(vmx->vcpu.kvm))
2472 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
ad312c7c 2473 page_to_phys(vmx->vcpu.arch.apic->regs_page));
f78e0e2e
SY		 2474 vmcs_write32(TPR_THRESHOLD, 0);
2475 }
2476
2477 if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm))
2478 vmcs_write64(APIC_ACCESS_ADDR,
bfc6d222 2479 page_to_phys(vmx->vcpu.kvm->arch.apic_access_page));
6aa8b732 2480
2384d2b3
SY		 2481 if (vmx->vpid != 0)
2482 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2483
ad312c7c
ZX		 2484 vmx->vcpu.arch.cr0 = 0x60000010;
2485 vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */
8b9cf98c 2486 vmx_set_cr4(&vmx->vcpu, 0);
8b9cf98c 2487 vmx_set_efer(&vmx->vcpu, 0);
8b9cf98c
RR		 2488 vmx_fpu_activate(&vmx->vcpu);
2489 update_exception_bitmap(&vmx->vcpu);
6aa8b732 2490
2384d2b3
SY		 2491 vpid_sync_vcpu_all(vmx);
2492
3200f405 2493 ret = 0;
6aa8b732 2494
a89a8fb9
MG		 2495 /* HACK: Don't enable emulation on guest boot/reset */
2496 vmx->emulation_required = 0;
2497
6aa8b732 2498out:
3200f405 2499 up_read(&vcpu->kvm->slots_lock);
6aa8b732
AK		 2500 return ret;
2501}
2502
3b86cd99
JK		 2503static void enable_irq_window(struct kvm_vcpu *vcpu)
2504{
2505 u32 cpu_based_vm_exec_control;
2506
2507 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2508 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
2509 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2510}
2511
2512static void enable_nmi_window(struct kvm_vcpu *vcpu)
2513{
2514 u32 cpu_based_vm_exec_control;
2515
2516 if (!cpu_has_virtual_nmis()) {
2517 enable_irq_window(vcpu);
2518 return;
2519 }
2520
2521 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
2522 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_NMI_PENDING;
2523 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2524}
2525
66fd3f7f 2526static void vmx_inject_irq(struct kvm_vcpu *vcpu)
85f455f7 2527{
9c8cba37 2528 struct vcpu_vmx *vmx = to_vmx(vcpu);
66fd3f7f
GN		 2529 uint32_t intr;
2530 int irq = vcpu->arch.interrupt.nr;
9c8cba37 2531
2714d1d3
FEL		 2532 KVMTRACE_1D(INJ_VIRQ, vcpu, (u32)irq, handler);
2533
fa89a817 2534 ++vcpu->stat.irq_injections;
56b237e3 2535 if (vcpu->arch.rmode.vm86_active) {
9c8cba37
AK		 2536 vmx->rmode.irq.pending = true;
2537 vmx->rmode.irq.vector = irq;
5fdbf976 2538 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
ae0bb3e0
GN		 2539 if (vcpu->arch.interrupt.soft)
2540 vmx->rmode.irq.rip +=
2541 vmx->vcpu.arch.event_exit_inst_len;
9c5623e3
AK		 2542 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2543 irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK);
2544 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
5fdbf976 2545 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
85f455f7
ED		 2546 return;
2547 }
66fd3f7f
GN		 2548 intr = irq | INTR_INFO_VALID_MASK;
2549 if (vcpu->arch.interrupt.soft) {
2550 intr |= INTR_TYPE_SOFT_INTR;
2551 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2552 vmx->vcpu.arch.event_exit_inst_len);
2553 } else
2554 intr |= INTR_TYPE_EXT_INTR;
2555 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
85f455f7
ED		 2556}
2557
f08864b4
SY		 2558static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
2559{
66a5a347
JK		 2560 struct vcpu_vmx *vmx = to_vmx(vcpu);
2561
3b86cd99
JK		 2562 if (!cpu_has_virtual_nmis()) {
2563 /*
2564 * Tracking the NMI-blocked state in software is built upon
2565 * finding the next open IRQ window. This, in turn, depends on
2566 * well-behaving guests: They have to keep IRQs disabled at
2567 * least as long as the NMI handler runs. Otherwise we may
2568 * cause NMI nesting, maybe breaking the guest. But as this is
2569 * highly unlikely, we can live with the residual risk.
2570 */
2571 vmx->soft_vnmi_blocked = 1;
2572 vmx->vnmi_blocked_time = 0;
2573 }
2574
487b391d 2575 ++vcpu->stat.nmi_injections;
56b237e3 2576 if (vcpu->arch.rmode.vm86_active) {
66a5a347
JK		 2577 vmx->rmode.irq.pending = true;
2578 vmx->rmode.irq.vector = NMI_VECTOR;
2579 vmx->rmode.irq.rip = kvm_rip_read(vcpu);
2580 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2581 NMI_VECTOR | INTR_TYPE_SOFT_INTR |
2582 INTR_INFO_VALID_MASK);
2583 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1);
2584 kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1);
2585 return;
2586 }
f08864b4
SY		 2587 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2588 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
f08864b4
SY		 2589}
2590
c4282df9 2591static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
33f089ca 2592{
3b86cd99 2593 if (!cpu_has_virtual_nmis() && to_vmx(vcpu)->soft_vnmi_blocked)
c4282df9 2594 return 0;
33f089ca 2595
c4282df9
GN		 2596 return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2597 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS |
2598 GUEST_INTR_STATE_NMI));
33f089ca
JK		 2599}
2600
78646121
GN		 2601static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
2602{
c4282df9
GN		 2603 return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
2604 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
2605 (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
78646121
GN		 2606}
2607
cbc94022
IE		 2608static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
2609{
2610 int ret;
2611 struct kvm_userspace_memory_region tss_mem = {
6fe63979 2612 .slot = TSS_PRIVATE_MEMSLOT,
cbc94022
IE		 2613 .guest_phys_addr = addr,
2614 .memory_size = PAGE_SIZE * 3,
2615 .flags = 0,
2616 };
2617
2618 ret = kvm_set_memory_region(kvm, &tss_mem, 0);
2619 if (ret)
2620 return ret;
bfc6d222 2621 kvm->arch.tss_addr = addr;
cbc94022
IE		 2622 return 0;
2623}
2624
6aa8b732
AK		 2625static int handle_rmode_exception(struct kvm_vcpu *vcpu,
2626 int vec, u32 err_code)
2627{
b3f37707
NK		 2628 /*
2629 * Instruction with address size override prefix opcode 0x67
2630 * Cause the #SS fault with 0 error code in VM86 mode.
2631 */
2632 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
3427318f 2633 if (emulate_instruction(vcpu, NULL, 0, 0, 0) == EMULATE_DONE)
6aa8b732 2634 return 1;
77ab6db0
JK		 2635 /*
2636 * Forward all other exceptions that are valid in real mode.
2637 * FIXME: Breaks guest debugging in real mode, needs to be fixed with
2638 * the required debugging infrastructure rework.
2639 */
2640 switch (vec) {
77ab6db0 2641 case DB_VECTOR:
d0bfb940
JK		 2642 if (vcpu->guest_debug &
2643 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
2644 return 0;
2645 kvm_queue_exception(vcpu, vec);
2646 return 1;
77ab6db0 2647 case BP_VECTOR:
d0bfb940
JK		 2648 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2649 return 0;
2650 /* fall through */
2651 case DE_VECTOR:
77ab6db0
JK		 2652 case OF_VECTOR:
2653 case BR_VECTOR:
2654 case UD_VECTOR:
2655 case DF_VECTOR:
2656 case SS_VECTOR:
2657 case GP_VECTOR:
2658 case MF_VECTOR:
2659 kvm_queue_exception(vcpu, vec);
2660 return 1;
2661 }
6aa8b732
AK		 2662 return 0;
2663}
2664
a0861c02
AK		 2665/*
2666 * Trigger machine check on the host. We assume all the MSRs are already set up
2667 * by the CPU and that we still run on the same CPU as the MCE occurred on.
2668 * We pass a fake environment to the machine check handler because we want
2669 * the guest to be always treated like user space, no matter what context
2670 * it used internally.
2671 */
2672static void kvm_machine_check(void)
2673{
2674#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_64)
2675 struct pt_regs regs = {
2676 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
2677 .flags = X86_EFLAGS_IF,
2678 };
2679
2680 do_machine_check(&regs, 0);
2681#endif
2682}
2683
2684static int handle_machine_check(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2685{
2686 /* already handled by vcpu_run */
2687 return 1;
2688}
2689
6aa8b732
AK		 2690static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2691{
1155f76a 2692 struct vcpu_vmx *vmx = to_vmx(vcpu);
d0bfb940 2693 u32 intr_info, ex_no, error_code;
42dbaa5a 2694 unsigned long cr2, rip, dr6;
6aa8b732
AK		 2695 u32 vect_info;
2696 enum emulation_result er;
2697
1155f76a 2698 vect_info = vmx->idt_vectoring_info;
6aa8b732
AK		 2699 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
2700
a0861c02
AK		 2701 if (is_machine_check(intr_info))
2702 return handle_machine_check(vcpu, kvm_run);
2703
6aa8b732 2704 if ((vect_info & VECTORING_INFO_VALID_MASK) &&
d77c26fc 2705 !is_page_fault(intr_info))
6aa8b732 2706 printk(KERN_ERR "%s: unexpected, vectoring info 0x%x "
b8688d51 2707 "intr info 0x%x\n", __func__, vect_info, intr_info);
6aa8b732 2708
e4a41889 2709 if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR)
1b6269db 2710 return 1; /* already handled by vmx_vcpu_run() */
2ab455cc
AL		 2711
2712 if (is_no_device(intr_info)) {
5fd86fcf 2713 vmx_fpu_activate(vcpu);
2ab455cc
AL		 2714 return 1;
2715 }
2716
7aa81cc0 2717 if (is_invalid_opcode(intr_info)) {
571008da 2718 er = emulate_instruction(vcpu, kvm_run, 0, 0, EMULTYPE_TRAP_UD);
7aa81cc0 2719 if (er != EMULATE_DONE)
7ee5d940 2720 kvm_queue_exception(vcpu, UD_VECTOR);
7aa81cc0
AL		 2721 return 1;
2722 }
2723
6aa8b732 2724 error_code = 0;
5fdbf976 2725 rip = kvm_rip_read(vcpu);
2e11384c 2726 if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
6aa8b732
AK		 2727 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
2728 if (is_page_fault(intr_info)) {
1439442c 2729 /* EPT won't cause page fault directly */
089d034e 2730 if (enable_ept)
1439442c 2731 BUG();
6aa8b732 2732 cr2 = vmcs_readl(EXIT_QUALIFICATION);
2714d1d3
FEL		 2733 KVMTRACE_3D(PAGE_FAULT, vcpu, error_code, (u32)cr2,
2734 (u32)((u64)cr2 >> 32), handler);
3298b75c 2735 if (kvm_event_needs_reinjection(vcpu))
577bdc49 2736 kvm_mmu_unprotect_page_virt(vcpu, cr2);
3067714c 2737 return kvm_mmu_page_fault(vcpu, cr2, error_code);
6aa8b732
AK		 2738 }
2739
56b237e3 2740 if (vcpu->arch.rmode.vm86_active &&
6aa8b732 2741 handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
72d6e5a0 2742 error_code)) {
ad312c7c
ZX		 2743 if (vcpu->arch.halt_request) {
2744 vcpu->arch.halt_request = 0;
72d6e5a0
AK		 2745 return kvm_emulate_halt(vcpu);
2746 }
6aa8b732 2747 return 1;
72d6e5a0 2748 }
6aa8b732 2749
d0bfb940 2750 ex_no = intr_info & INTR_INFO_VECTOR_MASK;
42dbaa5a
JK		 2751 switch (ex_no) {
2752 case DB_VECTOR:
2753 dr6 = vmcs_readl(EXIT_QUALIFICATION);
2754 if (!(vcpu->guest_debug &
2755 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
2756 vcpu->arch.dr6 = dr6 | DR6_FIXED_1;
2757 kvm_queue_exception(vcpu, DB_VECTOR);
2758 return 1;
2759 }
2760 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
2761 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
2762 /* fall through */
2763 case BP_VECTOR:
6aa8b732 2764 kvm_run->exit_reason = KVM_EXIT_DEBUG;
d0bfb940
JK		 2765 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
2766 kvm_run->debug.arch.exception = ex_no;
42dbaa5a
JK		 2767 break;
2768 default:
d0bfb940
JK		 2769 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
2770 kvm_run->ex.exception = ex_no;
2771 kvm_run->ex.error_code = error_code;
42dbaa5a 2772 break;
6aa8b732 2773 }
6aa8b732
AK		 2774 return 0;
2775}
2776
2777static int handle_external_interrupt(struct kvm_vcpu *vcpu,
2778 struct kvm_run *kvm_run)
2779{
1165f5fe 2780 ++vcpu->stat.irq_exits;
2714d1d3 2781 KVMTRACE_1D(INTR, vcpu, vmcs_read32(VM_EXIT_INTR_INFO), handler);
6aa8b732
AK		 2782 return 1;
2783}
2784
988ad74f
AK		 2785static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2786{
2787 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2788 return 0;
2789}
6aa8b732 2790
6aa8b732
AK		 2791static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2792{
bfdaab09 2793 unsigned long exit_qualification;
34c33d16 2794 int size, in, string;
039576c0 2795 unsigned port;
6aa8b732 2796
1165f5fe 2797 ++vcpu->stat.io_exits;
bfdaab09 2798 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
039576c0 2799 string = (exit_qualification & 16) != 0;
e70669ab
LV		 2800
2801 if (string) {
3427318f
LV		 2802 if (emulate_instruction(vcpu,
2803 kvm_run, 0, 0, 0) == EMULATE_DO_MMIO)
e70669ab
LV		 2804 return 0;
2805 return 1;
2806 }
2807
2808 size = (exit_qualification & 7) + 1;
2809 in = (exit_qualification & 8) != 0;
039576c0 2810 port = exit_qualification >> 16;
e70669ab 2811
e93f36bc 2812 skip_emulated_instruction(vcpu);
3090dd73 2813 return kvm_emulate_pio(vcpu, kvm_run, in, size, port);
6aa8b732
AK		 2814}
2815
102d8325
IM		 2816static void
2817vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
2818{
2819 /*
2820 * Patch in the VMCALL instruction:
2821 */
2822 hypercall[0] = 0x0f;
2823 hypercall[1] = 0x01;
2824 hypercall[2] = 0xc1;
102d8325
IM		 2825}
2826
6aa8b732
AK		 2827static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2828{
bfdaab09 2829 unsigned long exit_qualification;
6aa8b732
AK		 2830 int cr;
2831 int reg;
2832
bfdaab09 2833 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
6aa8b732
AK		 2834 cr = exit_qualification & 15;
2835 reg = (exit_qualification >> 8) & 15;
2836 switch ((exit_qualification >> 4) & 3) {
2837 case 0: /* mov to cr */
5fdbf976
MT		 2838 KVMTRACE_3D(CR_WRITE, vcpu, (u32)cr,
2839 (u32)kvm_register_read(vcpu, reg),
2840 (u32)((u64)kvm_register_read(vcpu, reg) >> 32),
2841 handler);
6aa8b732
AK		 2842 switch (cr) {
2843 case 0:
5fdbf976 2844 kvm_set_cr0(vcpu, kvm_register_read(vcpu, reg));
6aa8b732
AK		 2845 skip_emulated_instruction(vcpu);
2846 return 1;
2847 case 3:
5fdbf976 2848 kvm_set_cr3(vcpu, kvm_register_read(vcpu, reg));
6aa8b732
AK		 2849 skip_emulated_instruction(vcpu);
2850 return 1;
2851 case 4:
5fdbf976 2852 kvm_set_cr4(vcpu, kvm_register_read(vcpu, reg));
6aa8b732
AK		 2853 skip_emulated_instruction(vcpu);
2854 return 1;
0a5fff19
GN		 2855 case 8: {
2856 u8 cr8_prev = kvm_get_cr8(vcpu);
2857 u8 cr8 = kvm_register_read(vcpu, reg);
2858 kvm_set_cr8(vcpu, cr8);
2859 skip_emulated_instruction(vcpu);
2860 if (irqchip_in_kernel(vcpu->kvm))
2861 return 1;
2862 if (cr8_prev <= cr8)
2863 return 1;
2864 kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2865 return 0;
2866 }
6aa8b732
AK		 2867 };
2868 break;
25c4c276 2869 case 2: /* clts */
5fd86fcf 2870 vmx_fpu_deactivate(vcpu);
ad312c7c
ZX		 2871 vcpu->arch.cr0 &= ~X86_CR0_TS;
2872 vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0);
5fd86fcf 2873 vmx_fpu_activate(vcpu);
2714d1d3 2874 KVMTRACE_0D(CLTS, vcpu, handler);
25c4c276
AL		 2875 skip_emulated_instruction(vcpu);
2876 return 1;
6aa8b732
AK		 2877 case 1: /*mov from cr*/
2878 switch (cr) {
2879 case 3:
5fdbf976 2880 kvm_register_write(vcpu, reg, vcpu->arch.cr3);
2714d1d3 2881 KVMTRACE_3D(CR_READ, vcpu, (u32)cr,
5fdbf976
MT		 2882 (u32)kvm_register_read(vcpu, reg),
2883 (u32)((u64)kvm_register_read(vcpu, reg) >> 32),
2714d1d3 2884 handler);
6aa8b732
AK		 2885 skip_emulated_instruction(vcpu);
2886 return 1;
2887 case 8:
5fdbf976 2888 kvm_register_write(vcpu, reg, kvm_get_cr8(vcpu));
2714d1d3 2889 KVMTRACE_2D(CR_READ, vcpu, (u32)cr,
5fdbf976 2890 (u32)kvm_register_read(vcpu, reg), handler);
6aa8b732
AK		 2891 skip_emulated_instruction(vcpu);
2892 return 1;
2893 }
2894 break;
2895 case 3: /* lmsw */
2d3ad1f4 2896 kvm_lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f);
6aa8b732
AK		 2897
2898 skip_emulated_instruction(vcpu);
2899 return 1;
2900 default:
2901 break;
2902 }
2903 kvm_run->exit_reason = 0;
f0242478 2904 pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
6aa8b732
AK		 2905 (int)(exit_qualification >> 4) & 3, cr);
2906 return 0;
2907}
2908
2909static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2910{
bfdaab09 2911 unsigned long exit_qualification;
6aa8b732
AK		 2912 unsigned long val;
2913 int dr, reg;
2914
42dbaa5a
JK		 2915 dr = vmcs_readl(GUEST_DR7);
2916 if (dr & DR7_GD) {
2917 /*
2918 * As the vm-exit takes precedence over the debug trap, we
2919 * need to emulate the latter, either for the host or the
2920 * guest debugging itself.
2921 */
2922 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
2923 kvm_run->debug.arch.dr6 = vcpu->arch.dr6;
2924 kvm_run->debug.arch.dr7 = dr;
2925 kvm_run->debug.arch.pc =
2926 vmcs_readl(GUEST_CS_BASE) +
2927 vmcs_readl(GUEST_RIP);
2928 kvm_run->debug.arch.exception = DB_VECTOR;
2929 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2930 return 0;
2931 } else {
2932 vcpu->arch.dr7 &= ~DR7_GD;
2933 vcpu->arch.dr6 |= DR6_BD;
2934 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
2935 kvm_queue_exception(vcpu, DB_VECTOR);
2936 return 1;
2937 }
2938 }
2939
bfdaab09 2940 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
42dbaa5a
JK		 2941 dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
2942 reg = DEBUG_REG_ACCESS_REG(exit_qualification);
2943 if (exit_qualification & TYPE_MOV_FROM_DR) {
6aa8b732 2944 switch (dr) {
42dbaa5a
JK		 2945 case 0 ... 3:
2946 val = vcpu->arch.db[dr];
2947 break;
6aa8b732 2948 case 6:
42dbaa5a 2949 val = vcpu->arch.dr6;
6aa8b732
AK		 2950 break;
2951 case 7:
42dbaa5a 2952 val = vcpu->arch.dr7;
6aa8b732
AK		 2953 break;
2954 default:
2955 val = 0;
2956 }
5fdbf976 2957 kvm_register_write(vcpu, reg, val);
2714d1d3 2958 KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler);
6aa8b732 2959 } else {
42dbaa5a
JK		 2960 val = vcpu->arch.regs[reg];
2961 switch (dr) {
2962 case 0 ... 3:
2963 vcpu->arch.db[dr] = val;
2964 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
2965 vcpu->arch.eff_db[dr] = val;
2966 break;
2967 case 4 ... 5:
2968 if (vcpu->arch.cr4 & X86_CR4_DE)
2969 kvm_queue_exception(vcpu, UD_VECTOR);
2970 break;
2971 case 6:
2972 if (val & 0xffffffff00000000ULL) {
2973 kvm_queue_exception(vcpu, GP_VECTOR);
2974 break;
2975 }
2976 vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
2977 break;
2978 case 7:
2979 if (val & 0xffffffff00000000ULL) {
2980 kvm_queue_exception(vcpu, GP_VECTOR);
2981 break;
2982 }
2983 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
2984 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
2985 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
2986 vcpu->arch.switch_db_regs =
2987 (val & DR7_BP_EN_MASK);
2988 }
2989 break;
2990 }
2991 KVMTRACE_2D(DR_WRITE, vcpu, (u32)dr, (u32)val, handler);
6aa8b732 2992 }
6aa8b732
AK		 2993 skip_emulated_instruction(vcpu);
2994 return 1;
2995}
2996
2997static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2998{
06465c5a
AK		 2999 kvm_emulate_cpuid(vcpu);
3000 return 1;
6aa8b732
AK		 3001}
3002
3003static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3004{
ad312c7c 3005 u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
6aa8b732
AK		 3006 u64 data;
3007
3008 if (vmx_get_msr(vcpu, ecx, &data)) {
c1a5d4f9 3009 kvm_inject_gp(vcpu, 0);
6aa8b732
AK		 3010 return 1;
3011 }
3012
2714d1d3
FEL		 3013 KVMTRACE_3D(MSR_READ, vcpu, ecx, (u32)data, (u32)(data >> 32),
3014 handler);
3015
6aa8b732 3016 /* FIXME: handling of bits 32:63 of rax, rdx */
ad312c7c
ZX		 3017 vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u;
3018 vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
6aa8b732
AK		 3019 skip_emulated_instruction(vcpu);
3020 return 1;
3021}
3022
3023static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3024{
ad312c7c
ZX		 3025 u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
3026 u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
3027 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
6aa8b732 3028
2714d1d3
FEL		 3029 KVMTRACE_3D(MSR_WRITE, vcpu, ecx, (u32)data, (u32)(data >> 32),
3030 handler);
3031
6aa8b732 3032 if (vmx_set_msr(vcpu, ecx, data) != 0) {
c1a5d4f9 3033 kvm_inject_gp(vcpu, 0);
6aa8b732
AK		 3034 return 1;
3035 }
3036
3037 skip_emulated_instruction(vcpu);
3038 return 1;
3039}
3040
6e5d865c
YS		 3041static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu,
3042 struct kvm_run *kvm_run)
3043{
3044 return 1;
3045}
3046
6aa8b732
AK		 3047static int handle_interrupt_window(struct kvm_vcpu *vcpu,
3048 struct kvm_run *kvm_run)
3049{
85f455f7
ED		 3050 u32 cpu_based_vm_exec_control;
3051
3052 /* clear pending irq */
3053 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3054 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
3055 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
2714d1d3
FEL		 3056
3057 KVMTRACE_0D(PEND_INTR, vcpu, handler);
a26bf12a 3058 ++vcpu->stat.irq_window_exits;
2714d1d3 3059
c1150d8c
DL		 3060 /*
3061 * If the user space waits to inject interrupts, exit as soon as
3062 * possible
3063 */
8061823a
GN		 3064 if (!irqchip_in_kernel(vcpu->kvm) &&
3065 kvm_run->request_interrupt_window &&
3066 !kvm_cpu_has_interrupt(vcpu)) {
c1150d8c 3067 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
c1150d8c
DL		 3068 return 0;
3069 }
6aa8b732
AK		 3070 return 1;
3071}
3072
3073static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3074{
3075 skip_emulated_instruction(vcpu);
d3bef15f 3076 return kvm_emulate_halt(vcpu);
6aa8b732
AK		 3077}
3078
c21415e8
IM		 3079static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3080{
510043da 3081 skip_emulated_instruction(vcpu);
7aa81cc0
AL		 3082 kvm_emulate_hypercall(vcpu);
3083 return 1;
c21415e8
IM		 3084}
3085
e3c7cb6a
AK		 3086static int handle_vmx_insn(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3087{
3088 kvm_queue_exception(vcpu, UD_VECTOR);
3089 return 1;
3090}
3091
a7052897
MT		 3092static int handle_invlpg(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3093{
f9c617f6 3094 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
a7052897
MT		 3095
3096 kvm_mmu_invlpg(vcpu, exit_qualification);
3097 skip_emulated_instruction(vcpu);
3098 return 1;
3099}
3100
e5edaa01
ED		 3101static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3102{
3103 skip_emulated_instruction(vcpu);
3104 /* TODO: Add support for VT-d/pass-through device */
3105 return 1;
3106}
3107
f78e0e2e
SY		 3108static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3109{
f9c617f6 3110 unsigned long exit_qualification;
f78e0e2e
SY		 3111 enum emulation_result er;
3112 unsigned long offset;
3113
f9c617f6 3114 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
f78e0e2e
SY		 3115 offset = exit_qualification & 0xffful;
3116
3117 er = emulate_instruction(vcpu, kvm_run, 0, 0, 0);
3118
3119 if (er != EMULATE_DONE) {
3120 printk(KERN_ERR
3121 "Fail to handle apic access vmexit! Offset is 0x%lx\n",
3122 offset);
3123 return -ENOTSUPP;
3124 }
3125 return 1;
3126}
3127
37817f29
IE		 3128static int handle_task_switch(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3129{
60637aac 3130 struct vcpu_vmx *vmx = to_vmx(vcpu);
37817f29
IE		 3131 unsigned long exit_qualification;
3132 u16 tss_selector;
64a7ec06
GN		 3133 int reason, type, idt_v;
3134
3135 idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
3136 type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
37817f29
IE		 3137
3138 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
3139
3140 reason = (u32)exit_qualification >> 30;
64a7ec06
GN		 3141 if (reason == TASK_SWITCH_GATE && idt_v) {
3142 switch (type) {
3143 case INTR_TYPE_NMI_INTR:
3144 vcpu->arch.nmi_injected = false;
3145 if (cpu_has_virtual_nmis())
3146 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3147 GUEST_INTR_STATE_NMI);
3148 break;
3149 case INTR_TYPE_EXT_INTR:
66fd3f7f 3150 case INTR_TYPE_SOFT_INTR:
64a7ec06
GN		 3151 kvm_clear_interrupt_queue(vcpu);
3152 break;
3153 case INTR_TYPE_HARD_EXCEPTION:
3154 case INTR_TYPE_SOFT_EXCEPTION:
3155 kvm_clear_exception_queue(vcpu);
3156 break;
3157 default:
3158 break;
3159 }
60637aac 3160 }
37817f29
IE		 3161 tss_selector = exit_qualification;
3162
64a7ec06
GN		 3163 if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
3164 type != INTR_TYPE_EXT_INTR &&
3165 type != INTR_TYPE_NMI_INTR))
3166 skip_emulated_instruction(vcpu);
3167
42dbaa5a
JK		 3168 if (!kvm_task_switch(vcpu, tss_selector, reason))
3169 return 0;
3170
3171 /* clear all local breakpoint enable flags */
3172 vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55);
3173
3174 /*
3175 * TODO: What about debug traps on tss switch?
3176 * Are we supposed to inject them and update dr6?
3177 */
3178
3179 return 1;
37817f29
IE		 3180}
3181
1439442c
SY		 3182static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3183{
f9c617f6 3184 unsigned long exit_qualification;
1439442c 3185 gpa_t gpa;
1439442c 3186 int gla_validity;
1439442c 3187
f9c617f6 3188 exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
1439442c
SY		 3189
3190 if (exit_qualification & (1 << 6)) {
3191 printk(KERN_ERR "EPT: GPA exceeds GAW!\n");
3192 return -ENOTSUPP;
3193 }
3194
3195 gla_validity = (exit_qualification >> 7) & 0x3;
3196 if (gla_validity != 0x3 && gla_validity != 0x1 && gla_validity != 0) {
3197 printk(KERN_ERR "EPT: Handling EPT violation failed!\n");
3198 printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n",
3199 (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS),
f9c617f6 3200 vmcs_readl(GUEST_LINEAR_ADDRESS));
1439442c
SY		 3201 printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n",
3202 (long unsigned int)exit_qualification);
3203 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
596ae895
AK		 3204 kvm_run->hw.hardware_exit_reason = EXIT_REASON_EPT_VIOLATION;
3205 return 0;
1439442c
SY		 3206 }
3207
3208 gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
49cd7d22 3209 return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0);
1439442c
SY		 3210}
3211
f08864b4
SY		 3212static int handle_nmi_window(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3213{
3214 u32 cpu_based_vm_exec_control;
3215
3216 /* clear pending NMI */
3217 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
3218 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
3219 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
3220 ++vcpu->stat.nmi_window_exits;
3221
3222 return 1;
3223}
3224
ea953ef0
MG		 3225static void handle_invalid_guest_state(struct kvm_vcpu *vcpu,
3226 struct kvm_run *kvm_run)
3227{
8b3079a5
AK		 3228 struct vcpu_vmx *vmx = to_vmx(vcpu);
3229 enum emulation_result err = EMULATE_DONE;
ea953ef0 3230
ea953ef0 3231 local_irq_enable();
34f0c1ad 3232 preempt_enable();
ea953ef0
MG		 3233
3234 while (!guest_state_valid(vcpu)) {
3235 err = emulate_instruction(vcpu, kvm_run, 0, 0, 0);
3236
1d5a4d9b
GT		 3237 if (err == EMULATE_DO_MMIO)
3238 break;
3239
3240 if (err != EMULATE_DONE) {
3241 kvm_report_emulation_failure(vcpu, "emulation failure");
263799a3 3242 break;
ea953ef0
MG		 3243 }
3244
3245 if (signal_pending(current))
3246 break;
3247 if (need_resched())
3248 schedule();
3249 }
3250
ea953ef0 3251 preempt_disable();
34f0c1ad 3252 local_irq_disable();
8b3079a5
AK		 3253
3254 vmx->invalid_state_emulation_result = err;
ea953ef0
MG		 3255}
3256
6aa8b732
AK		 3257/*
3258 * The exit handlers return 1 if the exit was handled fully and guest execution
3259 * may resume. Otherwise they set the kvm_run parameter to indicate what needs
3260 * to be done to userspace and return 0.
3261 */
3262static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
3263 struct kvm_run *kvm_run) = {
3264 [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
3265 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
988ad74f 3266 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
f08864b4 3267 [EXIT_REASON_NMI_WINDOW] = handle_nmi_window,
6aa8b732 3268 [EXIT_REASON_IO_INSTRUCTION] = handle_io,
6aa8b732
AK		 3269 [EXIT_REASON_CR_ACCESS] = handle_cr,
3270 [EXIT_REASON_DR_ACCESS] = handle_dr,
3271 [EXIT_REASON_CPUID] = handle_cpuid,
3272 [EXIT_REASON_MSR_READ] = handle_rdmsr,
3273 [EXIT_REASON_MSR_WRITE] = handle_wrmsr,
3274 [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window,
3275 [EXIT_REASON_HLT] = handle_halt,
a7052897 3276 [EXIT_REASON_INVLPG] = handle_invlpg,
c21415e8 3277 [EXIT_REASON_VMCALL] = handle_vmcall,
e3c7cb6a
AK		 3278 [EXIT_REASON_VMCLEAR] = handle_vmx_insn,
3279 [EXIT_REASON_VMLAUNCH] = handle_vmx_insn,
3280 [EXIT_REASON_VMPTRLD] = handle_vmx_insn,
3281 [EXIT_REASON_VMPTRST] = handle_vmx_insn,
3282 [EXIT_REASON_VMREAD] = handle_vmx_insn,
3283 [EXIT_REASON_VMRESUME] = handle_vmx_insn,
3284 [EXIT_REASON_VMWRITE] = handle_vmx_insn,
3285 [EXIT_REASON_VMOFF] = handle_vmx_insn,
3286 [EXIT_REASON_VMON] = handle_vmx_insn,
f78e0e2e
SY		 3287 [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold,
3288 [EXIT_REASON_APIC_ACCESS] = handle_apic_access,
e5edaa01 3289 [EXIT_REASON_WBINVD] = handle_wbinvd,
37817f29 3290 [EXIT_REASON_TASK_SWITCH] = handle_task_switch,
1439442c 3291 [EXIT_REASON_EPT_VIOLATION] = handle_ept_violation,
a0861c02 3292 [EXIT_REASON_MCE_DURING_VMENTRY] = handle_machine_check,
6aa8b732
AK		 3293};
3294
3295static const int kvm_vmx_max_exit_handlers =
50a3485c 3296 ARRAY_SIZE(kvm_vmx_exit_handlers);
6aa8b732
AK		 3297
3298/*
3299 * The guest has exited. See if we can fix it or if we need userspace
3300 * assistance.
3301 */
6062d012 3302static int vmx_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
6aa8b732 3303{
29bd8a78 3304 struct vcpu_vmx *vmx = to_vmx(vcpu);
a0861c02 3305 u32 exit_reason = vmx->exit_reason;
1155f76a 3306 u32 vectoring_info = vmx->idt_vectoring_info;
29bd8a78 3307
5fdbf976
MT		 3308 KVMTRACE_3D(VMEXIT, vcpu, exit_reason, (u32)kvm_rip_read(vcpu),
3309 (u32)((u64)kvm_rip_read(vcpu) >> 32), entryexit);
2714d1d3 3310
1d5a4d9b
GT		 3311 /* If we need to emulate an MMIO from handle_invalid_guest_state
3312 * we just return 0 */
10f32d84
AK		 3313 if (vmx->emulation_required && emulate_invalid_guest_state) {
3314 if (guest_state_valid(vcpu))
3315 vmx->emulation_required = 0;
8b3079a5 3316 return vmx->invalid_state_emulation_result != EMULATE_DO_MMIO;
10f32d84 3317 }
1d5a4d9b 3318
1439442c
SY		 3319 /* Access CR3 don't cause VMExit in paging mode, so we need
3320 * to sync with guest real CR3. */
6de4f3ad 3321 if (enable_ept && is_paging(vcpu))
1439442c 3322 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
1439442c 3323
29bd8a78
AK		 3324 if (unlikely(vmx->fail)) {
3325 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3326 kvm_run->fail_entry.hardware_entry_failure_reason
3327 = vmcs_read32(VM_INSTRUCTION_ERROR);
3328 return 0;
3329 }
6aa8b732 3330
d77c26fc 3331 if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
1439442c 3332 (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
60637aac
JK		 3333 exit_reason != EXIT_REASON_EPT_VIOLATION &&
3334 exit_reason != EXIT_REASON_TASK_SWITCH))
3335 printk(KERN_WARNING "%s: unexpected, valid vectoring info "
3336 "(0x%x) and exit reason is 0x%x\n",
3337 __func__, vectoring_info, exit_reason);
3b86cd99
JK		 3338
3339 if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked)) {
c4282df9 3340 if (vmx_interrupt_allowed(vcpu)) {
3b86cd99 3341 vmx->soft_vnmi_blocked = 0;
3b86cd99 3342 } else if (vmx->vnmi_blocked_time > 1000000000LL &&
4531220b 3343 vcpu->arch.nmi_pending) {
3b86cd99
JK		 3344 /*
3345 * This CPU don't support us in finding the end of an
3346 * NMI-blocked window if the guest runs with IRQs
3347 * disabled. So we pull the trigger after 1 s of
3348 * futile waiting, but inform the user about this.
3349 */
3350 printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
3351 "state on VCPU %d after 1 s timeout\n",
3352 __func__, vcpu->vcpu_id);
3353 vmx->soft_vnmi_blocked = 0;
3b86cd99 3354 }
3b86cd99
JK		 3355 }
3356
6aa8b732
AK		 3357 if (exit_reason < kvm_vmx_max_exit_handlers
3358 && kvm_vmx_exit_handlers[exit_reason])
3359 return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run);
3360 else {
3361 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
3362 kvm_run->hw.hardware_exit_reason = exit_reason;
3363 }
3364 return 0;
3365}
3366
95ba8273 3367static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
6e5d865c 3368{
95ba8273 3369 if (irr == -1 || tpr < irr) {
6e5d865c
YS		 3370 vmcs_write32(TPR_THRESHOLD, 0);
3371 return;
3372 }
3373
95ba8273 3374 vmcs_write32(TPR_THRESHOLD, irr);
6e5d865c
YS		 3375}
3376
cf393f75
AK		 3377static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
3378{
3379 u32 exit_intr_info;
7b4a25cb 3380 u32 idt_vectoring_info = vmx->idt_vectoring_info;
cf393f75
AK		 3381 bool unblock_nmi;
3382 u8 vector;
668f612f
AK		 3383 int type;
3384 bool idtv_info_valid;
cf393f75
AK		 3385
3386 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
20f65983 3387
a0861c02
AK		 3388 vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
3389
3390 /* Handle machine checks before interrupts are enabled */
3391 if ((vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
3392 || (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI
3393 && is_machine_check(exit_intr_info)))
3394 kvm_machine_check();
3395
20f65983
GN		 3396 /* We need to handle NMIs before interrupts are enabled */
3397 if ((exit_intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR &&
3398 (exit_intr_info & INTR_INFO_VALID_MASK)) {
3399 KVMTRACE_0D(NMI, &vmx->vcpu, handler);
3400 asm("int $2");
3401 }
3402
3403 idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
3404
cf393f75
AK		 3405 if (cpu_has_virtual_nmis()) {
3406 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
3407 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
3408 /*
7b4a25cb 3409 * SDM 3: 27.7.1.2 (September 2008)
cf393f75
AK		 3410 * Re-set bit "block by NMI" before VM entry if vmexit caused by
3411 * a guest IRET fault.
7b4a25cb
GN		 3412 * SDM 3: 23.2.2 (September 2008)
3413 * Bit 12 is undefined in any of the following cases:
3414 * If the VM exit sets the valid bit in the IDT-vectoring
3415 * information field.
3416 * If the VM exit is due to a double fault.
cf393f75 3417 */
7b4a25cb
GN		 3418 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
3419 vector != DF_VECTOR && !idtv_info_valid)
cf393f75
AK		 3420 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
3421 GUEST_INTR_STATE_NMI);
3b86cd99
JK		 3422 } else if (unlikely(vmx->soft_vnmi_blocked))
3423 vmx->vnmi_blocked_time +=
3424 ktime_to_ns(ktime_sub(ktime_get(), vmx->entry_time));
668f612f 3425
37b96e98
GN		 3426 vmx->vcpu.arch.nmi_injected = false;
3427 kvm_clear_exception_queue(&vmx->vcpu);
3428 kvm_clear_interrupt_queue(&vmx->vcpu);
3429
3430 if (!idtv_info_valid)
3431 return;
3432
668f612f
AK		 3433 vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
3434 type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
37b96e98 3435
64a7ec06 3436 switch (type) {
37b96e98
GN		 3437 case INTR_TYPE_NMI_INTR:
3438 vmx->vcpu.arch.nmi_injected = true;
668f612f 3439 /*
7b4a25cb 3440 * SDM 3: 27.7.1.2 (September 2008)
37b96e98
GN		 3441 * Clear bit "block by NMI" before VM entry if a NMI
3442 * delivery faulted.
668f612f 3443 */
37b96e98
GN		 3444 vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
3445 GUEST_INTR_STATE_NMI);
3446 break;
37b96e98 3447 case INTR_TYPE_SOFT_EXCEPTION:
66fd3f7f
GN		 3448 vmx->vcpu.arch.event_exit_inst_len =
3449 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3450 /* fall through */
3451 case INTR_TYPE_HARD_EXCEPTION:
35920a35 3452 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
37b96e98
GN		 3453 u32 err = vmcs_read32(IDT_VECTORING_ERROR_CODE);
3454 kvm_queue_exception_e(&vmx->vcpu, vector, err);
35920a35
AK		 3455 } else
3456 kvm_queue_exception(&vmx->vcpu, vector);
37b96e98 3457 break;
66fd3f7f
GN		 3458 case INTR_TYPE_SOFT_INTR:
3459 vmx->vcpu.arch.event_exit_inst_len =
3460 vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3461 /* fall through */
37b96e98 3462 case INTR_TYPE_EXT_INTR:
66fd3f7f
GN		 3463 kvm_queue_interrupt(&vmx->vcpu, vector,
3464 type == INTR_TYPE_SOFT_INTR);
37b96e98
GN		 3465 break;
3466 default:
3467 break;
f7d9238f 3468 }
cf393f75
AK		 3469}
3470
9c8cba37
AK		 3471/*
3472 * Failure to inject an interrupt should give us the information
3473 * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs
3474 * when fetching the interrupt redirection bitmap in the real-mode
3475 * tss, this doesn't happen. So we do it ourselves.
3476 */
3477static void fixup_rmode_irq(struct vcpu_vmx *vmx)
3478{
3479 vmx->rmode.irq.pending = 0;
5fdbf976 3480 if (kvm_rip_read(&vmx->vcpu) + 1 != vmx->rmode.irq.rip)
9c8cba37 3481 return;
5fdbf976 3482 kvm_rip_write(&vmx->vcpu, vmx->rmode.irq.rip);
9c8cba37
AK		 3483 if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) {
3484 vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK;
3485 vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR;
3486 return;
3487 }
3488 vmx->idt_vectoring_info =
3489 VECTORING_INFO_VALID_MASK
3490 | INTR_TYPE_EXT_INTR
3491 | vmx->rmode.irq.vector;
3492}
3493
c801949d
AK		 3494#ifdef CONFIG_X86_64
3495#define R "r"
3496#define Q "q"
3497#else
3498#define R "e"
3499#define Q "l"
3500#endif
3501
04d2cc77 3502static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
6aa8b732 3503{
a2fa3e9f 3504 struct vcpu_vmx *vmx = to_vmx(vcpu);
e6adf283 3505
8f5d549f
AK		 3506 if (enable_ept && is_paging(vcpu)) {
3507 vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
3508 ept_load_pdptrs(vcpu);
3509 }
3b86cd99
JK		 3510 /* Record the guest's net vcpu time for enforced NMI injections. */
3511 if (unlikely(!cpu_has_virtual_nmis() && vmx->soft_vnmi_blocked))
3512 vmx->entry_time = ktime_get();
3513
a89a8fb9
MG		 3514 /* Handle invalid guest state instead of entering VMX */
3515 if (vmx->emulation_required && emulate_invalid_guest_state) {
3516 handle_invalid_guest_state(vcpu, kvm_run);
3517 return;
3518 }
3519
5fdbf976
MT		 3520 if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
3521 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
3522 if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
3523 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
3524
787ff736
GN		 3525 /* When single-stepping over STI and MOV SS, we must clear the
3526 * corresponding interruptibility bits in the guest state. Otherwise
3527 * vmentry fails as it then expects bit 14 (BS) in pending debug
3528 * exceptions being set, but that's not correct for the guest debugging
3529 * case. */
3530 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
3531 vmx_set_interrupt_shadow(vcpu, 0);
3532
e6adf283
AK		 3533 /*
3534 * Loading guest fpu may have cleared host cr0.ts
3535 */
3536 vmcs_writel(HOST_CR0, read_cr0());
3537
42dbaa5a
JK		 3538 set_debugreg(vcpu->arch.dr6, 6);
3539
d77c26fc 3540 asm(
6aa8b732 3541 /* Store host registers */
c801949d
AK		 3542 "push %%"R"dx; push %%"R"bp;"
3543 "push %%"R"cx \n\t"
313dbd49
AK		 3544 "cmp %%"R"sp, %c[host_rsp](%0) \n\t"
3545 "je 1f \n\t"
3546 "mov %%"R"sp, %c[host_rsp](%0) \n\t"
4ecac3fd 3547 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
313dbd49 3548 "1: \n\t"
6aa8b732 3549 /* Check if vmlaunch of vmresume is needed */
e08aa78a 3550 "cmpl $0, %c[launched](%0) \n\t"
6aa8b732 3551 /* Load guest registers. Don't clobber flags. */
c801949d
AK		 3552 "mov %c[cr2](%0), %%"R"ax \n\t"
3553 "mov %%"R"ax, %%cr2 \n\t"
3554 "mov %c[rax](%0), %%"R"ax \n\t"
3555 "mov %c[rbx](%0), %%"R"bx \n\t"
3556 "mov %c[rdx](%0), %%"R"dx \n\t"
3557 "mov %c[rsi](%0), %%"R"si \n\t"
3558 "mov %c[rdi](%0), %%"R"di \n\t"
3559 "mov %c[rbp](%0), %%"R"bp \n\t"
05b3e0c2 3560#ifdef CONFIG_X86_64
e08aa78a
AK		 3561 "mov %c[r8](%0), %%r8 \n\t"
3562 "mov %c[r9](%0), %%r9 \n\t"
3563 "mov %c[r10](%0), %%r10 \n\t"
3564 "mov %c[r11](%0), %%r11 \n\t"
3565 "mov %c[r12](%0), %%r12 \n\t"
3566 "mov %c[r13](%0), %%r13 \n\t"
3567 "mov %c[r14](%0), %%r14 \n\t"
3568 "mov %c[r15](%0), %%r15 \n\t"
6aa8b732 3569#endif
c801949d
AK		 3570 "mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */
3571
6aa8b732 3572 /* Enter guest mode */
cd2276a7 3573 "jne .Llaunched \n\t"
4ecac3fd 3574 __ex(ASM_VMX_VMLAUNCH) "\n\t"
cd2276a7 3575 "jmp .Lkvm_vmx_return \n\t"
4ecac3fd 3576 ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
cd2276a7 3577 ".Lkvm_vmx_return: "
6aa8b732 3578 /* Save guest registers, load host registers, keep flags */
c801949d
AK		 3579 "xchg %0, (%%"R"sp) \n\t"
3580 "mov %%"R"ax, %c[rax](%0) \n\t"
3581 "mov %%"R"bx, %c[rbx](%0) \n\t"
3582 "push"Q" (%%"R"sp); pop"Q" %c[rcx](%0) \n\t"
3583 "mov %%"R"dx, %c[rdx](%0) \n\t"
3584 "mov %%"R"si, %c[rsi](%0) \n\t"
3585 "mov %%"R"di, %c[rdi](%0) \n\t"
3586 "mov %%"R"bp, %c[rbp](%0) \n\t"
05b3e0c2 3587#ifdef CONFIG_X86_64
e08aa78a
AK		 3588 "mov %%r8, %c[r8](%0) \n\t"
3589 "mov %%r9, %c[r9](%0) \n\t"
3590 "mov %%r10, %c[r10](%0) \n\t"
3591 "mov %%r11, %c[r11](%0) \n\t"
3592 "mov %%r12, %c[r12](%0) \n\t"
3593 "mov %%r13, %c[r13](%0) \n\t"
3594 "mov %%r14, %c[r14](%0) \n\t"
3595 "mov %%r15, %c[r15](%0) \n\t"
6aa8b732 3596#endif
c801949d
AK		 3597 "mov %%cr2, %%"R"ax \n\t"
3598 "mov %%"R"ax, %c[cr2](%0) \n\t"
3599
3600 "pop %%"R"bp; pop %%"R"bp; pop %%"R"dx \n\t"
e08aa78a
AK		 3601 "setbe %c[fail](%0) \n\t"
3602 : : "c"(vmx), "d"((unsigned long)HOST_RSP),
3603 [launched]"i"(offsetof(struct vcpu_vmx, launched)),
3604 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
313dbd49 3605 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
ad312c7c
ZX		 3606 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
3607 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
3608 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
3609 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
3610 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
3611 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
3612 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
05b3e0c2 3613#ifdef CONFIG_X86_64
ad312c7c
ZX		 3614 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
3615 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
3616 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
3617 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
3618 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
3619 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
3620 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
3621 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
6aa8b732 3622#endif
ad312c7c 3623 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2))
c2036300 3624 : "cc", "memory"
c801949d 3625 , R"bx", R"di", R"si"
c2036300 3626#ifdef CONFIG_X86_64
c2036300
LV		 3627 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
3628#endif
3629 );
6aa8b732 3630
6de4f3ad
AK		 3631 vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
3632 | (1 << VCPU_EXREG_PDPTR));
5fdbf976
MT		 3633 vcpu->arch.regs_dirty = 0;
3634
42dbaa5a
JK		 3635 get_debugreg(vcpu->arch.dr6, 6);
3636
1155f76a 3637 vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
9c8cba37
AK		 3638 if (vmx->rmode.irq.pending)
3639 fixup_rmode_irq(vmx);
1155f76a 3640
d77c26fc 3641 asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
15ad7146 3642 vmx->launched = 1;
1b6269db 3643
cf393f75 3644 vmx_complete_interrupts(vmx);
6aa8b732
AK		 3645}
3646
c801949d
AK		 3647#undef R
3648#undef Q
3649
6aa8b732
AK		 3650static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
3651{
a2fa3e9f
GH		 3652 struct vcpu_vmx *vmx = to_vmx(vcpu);
3653
3654 if (vmx->vmcs) {
543e4243 3655 vcpu_clear(vmx);
a2fa3e9f
GH		 3656 free_vmcs(vmx->vmcs);
3657 vmx->vmcs = NULL;
6aa8b732
AK		 3658 }
3659}
3660
3661static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
3662{
fb3f0f51
RR		 3663 struct vcpu_vmx *vmx = to_vmx(vcpu);
3664
2384d2b3
SY		 3665 spin_lock(&vmx_vpid_lock);
3666 if (vmx->vpid != 0)
3667 __clear_bit(vmx->vpid, vmx_vpid_bitmap);
3668 spin_unlock(&vmx_vpid_lock);
6aa8b732 3669 vmx_free_vmcs(vcpu);
fb3f0f51
RR		 3670 kfree(vmx->host_msrs);
3671 kfree(vmx->guest_msrs);
3672 kvm_vcpu_uninit(vcpu);
a4770347 3673 kmem_cache_free(kvm_vcpu_cache, vmx);
6aa8b732
AK		 3674}
3675
fb3f0f51 3676static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
6aa8b732 3677{
fb3f0f51 3678 int err;
c16f862d 3679 struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
15ad7146 3680 int cpu;
6aa8b732 3681
a2fa3e9f 3682 if (!vmx)
fb3f0f51
RR		 3683 return ERR_PTR(-ENOMEM);
3684
2384d2b3
SY		 3685 allocate_vpid(vmx);
3686
fb3f0f51
RR		 3687 err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
3688 if (err)
3689 goto free_vcpu;
965b58a5 3690
a2fa3e9f 3691 vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
fb3f0f51
RR		 3692 if (!vmx->guest_msrs) {
3693 err = -ENOMEM;
3694 goto uninit_vcpu;
3695 }
965b58a5 3696
a2fa3e9f
GH		 3697 vmx->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
3698 if (!vmx->host_msrs)
fb3f0f51 3699 goto free_guest_msrs;
965b58a5 3700
a2fa3e9f
GH		 3701 vmx->vmcs = alloc_vmcs();
3702 if (!vmx->vmcs)
fb3f0f51 3703 goto free_msrs;
a2fa3e9f
GH		 3704
3705 vmcs_clear(vmx->vmcs);
3706
15ad7146
AK		 3707 cpu = get_cpu();
3708 vmx_vcpu_load(&vmx->vcpu, cpu);
8b9cf98c 3709 err = vmx_vcpu_setup(vmx);
fb3f0f51 3710 vmx_vcpu_put(&vmx->vcpu);
15ad7146 3711 put_cpu();
fb3f0f51
RR		 3712 if (err)
3713 goto free_vmcs;
5e4a0b3c
MT		 3714 if (vm_need_virtualize_apic_accesses(kvm))
3715 if (alloc_apic_access_page(kvm) != 0)
3716 goto free_vmcs;
fb3f0f51 3717
089d034e 3718 if (enable_ept)
b7ebfb05
SY		 3719 if (alloc_identity_pagetable(kvm) != 0)
3720 goto free_vmcs;
3721
fb3f0f51
RR		 3722 return &vmx->vcpu;
3723
3724free_vmcs:
3725 free_vmcs(vmx->vmcs);
3726free_msrs:
3727 kfree(vmx->host_msrs);
3728free_guest_msrs:
3729 kfree(vmx->guest_msrs);
3730uninit_vcpu:
3731 kvm_vcpu_uninit(&vmx->vcpu);
3732free_vcpu:
a4770347 3733 kmem_cache_free(kvm_vcpu_cache, vmx);
fb3f0f51 3734 return ERR_PTR(err);
6aa8b732
AK		 3735}
3736
002c7f7c
YS		 3737static void __init vmx_check_processor_compat(void *rtn)
3738{
3739 struct vmcs_config vmcs_conf;
3740
3741 *(int *)rtn = 0;
3742 if (setup_vmcs_config(&vmcs_conf) < 0)
3743 *(int *)rtn = -EIO;
3744 if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
3745 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
3746 smp_processor_id());
3747 *(int *)rtn = -EIO;
3748 }
3749}
3750
67253af5
SY		 3751static int get_ept_level(void)
3752{
3753 return VMX_EPT_DEFAULT_GAW + 1;
3754}
3755
4b12f0de 3756static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
64d4d521 3757{
4b12f0de
SY		 3758 u64 ret;
3759
522c68c4
SY		 3760 /* For VT-d and EPT combination
3761 * 1. MMIO: always map as UC
3762 * 2. EPT with VT-d:
3763 * a. VT-d without snooping control feature: can't guarantee the
3764 * result, try to trust guest.
3765 * b. VT-d with snooping control feature: snooping control feature of
3766 * VT-d engine can guarantee the cache correctness. Just set it
3767 * to WB to keep consistent with host. So the same as item 3.
3768 * 3. EPT without VT-d: always map as WB and set IGMT=1 to keep
3769 * consistent with host MTRR
3770 */
4b12f0de
SY		 3771 if (is_mmio)
3772 ret = MTRR_TYPE_UNCACHABLE << VMX_EPT_MT_EPTE_SHIFT;
522c68c4
SY		 3773 else if (vcpu->kvm->arch.iommu_domain &&
3774 !(vcpu->kvm->arch.iommu_flags & KVM_IOMMU_CACHE_COHERENCY))
3775 ret = kvm_get_guest_memory_type(vcpu, gfn) <<
3776 VMX_EPT_MT_EPTE_SHIFT;
4b12f0de 3777 else
522c68c4
SY		 3778 ret = (MTRR_TYPE_WRBACK << VMX_EPT_MT_EPTE_SHIFT)
3779 | VMX_EPT_IGMT_BIT;
4b12f0de
SY		 3780
3781 return ret;
64d4d521
SY		 3782}
3783
cbdd1bea 3784static struct kvm_x86_ops vmx_x86_ops = {
6aa8b732
AK		 3785 .cpu_has_kvm_support = cpu_has_kvm_support,
3786 .disabled_by_bios = vmx_disabled_by_bios,
3787 .hardware_setup = hardware_setup,
3788 .hardware_unsetup = hardware_unsetup,
002c7f7c 3789 .check_processor_compatibility = vmx_check_processor_compat,
6aa8b732
AK		 3790 .hardware_enable = hardware_enable,
3791 .hardware_disable = hardware_disable,
04547156 3792 .cpu_has_accelerated_tpr = report_flexpriority,
6aa8b732
AK		 3793
3794 .vcpu_create = vmx_create_vcpu,
3795 .vcpu_free = vmx_free_vcpu,
04d2cc77 3796 .vcpu_reset = vmx_vcpu_reset,
6aa8b732 3797
04d2cc77 3798 .prepare_guest_switch = vmx_save_host_state,
6aa8b732
AK		 3799 .vcpu_load = vmx_vcpu_load,
3800 .vcpu_put = vmx_vcpu_put,
3801
3802 .set_guest_debug = set_guest_debug,
3803 .get_msr = vmx_get_msr,
3804 .set_msr = vmx_set_msr,
3805 .get_segment_base = vmx_get_segment_base,
3806 .get_segment = vmx_get_segment,
3807 .set_segment = vmx_set_segment,
2e4d2653 3808 .get_cpl = vmx_get_cpl,
6aa8b732 3809 .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
25c4c276 3810 .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
6aa8b732 3811 .set_cr0 = vmx_set_cr0,
6aa8b732
AK		 3812 .set_cr3 = vmx_set_cr3,
3813 .set_cr4 = vmx_set_cr4,
6aa8b732 3814 .set_efer = vmx_set_efer,
6aa8b732
AK		 3815 .get_idt = vmx_get_idt,
3816 .set_idt = vmx_set_idt,
3817 .get_gdt = vmx_get_gdt,
3818 .set_gdt = vmx_set_gdt,
5fdbf976 3819 .cache_reg = vmx_cache_reg,
6aa8b732
AK		 3820 .get_rflags = vmx_get_rflags,
3821 .set_rflags = vmx_set_rflags,
3822
3823 .tlb_flush = vmx_flush_tlb,
6aa8b732 3824
6aa8b732 3825 .run = vmx_vcpu_run,
6062d012 3826 .handle_exit = vmx_handle_exit,
6aa8b732 3827 .skip_emulated_instruction = skip_emulated_instruction,
2809f5d2
GC		 3828 .set_interrupt_shadow = vmx_set_interrupt_shadow,
3829 .get_interrupt_shadow = vmx_get_interrupt_shadow,
102d8325 3830 .patch_hypercall = vmx_patch_hypercall,
2a8067f1 3831 .set_irq = vmx_inject_irq,
95ba8273 3832 .set_nmi = vmx_inject_nmi,
298101da 3833 .queue_exception = vmx_queue_exception,
78646121 3834 .interrupt_allowed = vmx_interrupt_allowed,
95ba8273
GN		 3835 .nmi_allowed = vmx_nmi_allowed,
3836 .enable_nmi_window = enable_nmi_window,
3837 .enable_irq_window = enable_irq_window,
3838 .update_cr8_intercept = update_cr8_intercept,
95ba8273 3839
cbc94022 3840 .set_tss_addr = vmx_set_tss_addr,
67253af5 3841 .get_tdp_level = get_ept_level,
4b12f0de 3842 .get_mt_mask = vmx_get_mt_mask,
6aa8b732
AK		 3843};
3844
3845static int __init vmx_init(void)
3846{
fdef3ad1
HQ		 3847 int r;
3848
3e7c73e9 3849 vmx_io_bitmap_a = (unsigned long *)__get_free_page(GFP_KERNEL);
fdef3ad1
HQ		 3850 if (!vmx_io_bitmap_a)
3851 return -ENOMEM;
3852
3e7c73e9 3853 vmx_io_bitmap_b = (unsigned long *)__get_free_page(GFP_KERNEL);
fdef3ad1
HQ		 3854 if (!vmx_io_bitmap_b) {
3855 r = -ENOMEM;
3856 goto out;
3857 }
3858
5897297b
AK		 3859 vmx_msr_bitmap_legacy = (unsigned long *)__get_free_page(GFP_KERNEL);
3860 if (!vmx_msr_bitmap_legacy) {
25c5f225
SY		 3861 r = -ENOMEM;
3862 goto out1;
3863 }
3864
5897297b
AK		 3865 vmx_msr_bitmap_longmode = (unsigned long *)__get_free_page(GFP_KERNEL);
3866 if (!vmx_msr_bitmap_longmode) {
3867 r = -ENOMEM;
3868 goto out2;
3869 }
3870
fdef3ad1
HQ		 3871 /*
3872 * Allow direct access to the PC debug port (it is often used for I/O
3873 * delays, but the vmexits simply slow things down).
3874 */
3e7c73e9
AK		 3875 memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
3876 clear_bit(0x80, vmx_io_bitmap_a);
fdef3ad1 3877
3e7c73e9 3878 memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
fdef3ad1 3879
5897297b
AK		 3880 memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
3881 memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
25c5f225 3882
2384d2b3
SY		 3883 set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
3884
cb498ea2 3885 r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE);
fdef3ad1 3886 if (r)
5897297b 3887 goto out3;
25c5f225 3888
5897297b
AK		 3889 vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
3890 vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
3891 vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
3892 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
3893 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
3894 vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
fdef3ad1 3895
089d034e 3896 if (enable_ept) {
1439442c 3897 bypass_guest_pf = 0;
5fdbcb9d 3898 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
2aaf69dc 3899 VMX_EPT_WRITABLE_MASK);
534e38b4 3900 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
4b12f0de 3901 VMX_EPT_EXECUTABLE_MASK);
5fdbcb9d
SY		 3902 kvm_enable_tdp();
3903 } else
3904 kvm_disable_tdp();
1439442c 3905
c7addb90
AK		 3906 if (bypass_guest_pf)
3907 kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull);
3908
1439442c
SY		 3909 ept_sync_global();
3910
fdef3ad1
HQ		 3911 return 0;
3912
5897297b
AK		 3913out3:
3914 free_page((unsigned long)vmx_msr_bitmap_longmode);
25c5f225 3915out2:
5897297b 3916 free_page((unsigned long)vmx_msr_bitmap_legacy);
fdef3ad1 3917out1:
3e7c73e9 3918 free_page((unsigned long)vmx_io_bitmap_b);
fdef3ad1 3919out:
3e7c73e9 3920 free_page((unsigned long)vmx_io_bitmap_a);
fdef3ad1 3921 return r;
6aa8b732
AK		 3922}
3923
3924static void __exit vmx_exit(void)
3925{
5897297b
AK		 3926 free_page((unsigned long)vmx_msr_bitmap_legacy);
3927 free_page((unsigned long)vmx_msr_bitmap_longmode);
3e7c73e9
AK		 3928 free_page((unsigned long)vmx_io_bitmap_b);
3929 free_page((unsigned long)vmx_io_bitmap_a);
fdef3ad1 3930
cb498ea2 3931 kvm_exit();
6aa8b732
AK		 3932}
3933
3934module_init(vmx_init)
3935module_exit(vmx_exit)
kernel source for telekom puls (alcatel/mediatek)