projects / GitHub / exynos8895 / android_kernel_samsung_universal8895.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: VMX: Move vm entry failure handling to the exit handler
[GitHub/exynos8895/android_kernel_samsung_universal8895.git] / drivers / kvm / kvm_main.c
CommitLineData
6aa8b732
AK		 1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
18#include "kvm.h"
e495606d
AK		 19#include "x86_emulate.h"
20#include "segment_descriptor.h"
85f455f7 21#include "irq.h"
6aa8b732
AK		 22
23#include <linux/kvm.h>
24#include <linux/module.h>
25#include <linux/errno.h>
6aa8b732
AK		 26#include <linux/percpu.h>
27#include <linux/gfp.h>
6aa8b732
AK		 28#include <linux/mm.h>
29#include <linux/miscdevice.h>
30#include <linux/vmalloc.h>
6aa8b732 31#include <linux/reboot.h>
6aa8b732
AK		 32#include <linux/debugfs.h>
33#include <linux/highmem.h>
34#include <linux/file.h>
59ae6c6b 35#include <linux/sysdev.h>
774c47f1 36#include <linux/cpu.h>
e8edc6e0 37#include <linux/sched.h>
d9e368d6
AK		 38#include <linux/cpumask.h>
39#include <linux/smp.h>
d6d28168 40#include <linux/anon_inodes.h>
6aa8b732 41
e495606d
AK		 42#include <asm/processor.h>
43#include <asm/msr.h>
44#include <asm/io.h>
45#include <asm/uaccess.h>
46#include <asm/desc.h>
6aa8b732
AK		 47
48MODULE_AUTHOR("Qumranet");
49MODULE_LICENSE("GPL");
50
133de902
AK		 51static DEFINE_SPINLOCK(kvm_lock);
52static LIST_HEAD(vm_list);
53
1b6c0168
AK		 54static cpumask_t cpus_hardware_enabled;
55
cbdd1bea 56struct kvm_x86_ops *kvm_x86_ops;
c16f862d
RR		 57struct kmem_cache *kvm_vcpu_cache;
58EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
1165f5fe 59
15ad7146
AK		 60static __read_mostly struct preempt_ops kvm_preempt_ops;
61
1165f5fe 62#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
6aa8b732
AK		 63
64static struct kvm_stats_debugfs_item {
65 const char *name;
1165f5fe 66 int offset;
6aa8b732
AK		 67 struct dentry *dentry;
68} debugfs_entries[] = {
1165f5fe
AK		 69 { "pf_fixed", STAT_OFFSET(pf_fixed) },
70 { "pf_guest", STAT_OFFSET(pf_guest) },
71 { "tlb_flush", STAT_OFFSET(tlb_flush) },
72 { "invlpg", STAT_OFFSET(invlpg) },
73 { "exits", STAT_OFFSET(exits) },
74 { "io_exits", STAT_OFFSET(io_exits) },
75 { "mmio_exits", STAT_OFFSET(mmio_exits) },
76 { "signal_exits", STAT_OFFSET(signal_exits) },
77 { "irq_window", STAT_OFFSET(irq_window_exits) },
78 { "halt_exits", STAT_OFFSET(halt_exits) },
b6958ce4 79 { "halt_wakeup", STAT_OFFSET(halt_wakeup) },
1165f5fe
AK		 80 { "request_irq", STAT_OFFSET(request_irq_exits) },
81 { "irq_exits", STAT_OFFSET(irq_exits) },
e6adf283 82 { "light_exits", STAT_OFFSET(light_exits) },
2cc51560 83 { "efer_reload", STAT_OFFSET(efer_reload) },
1165f5fe 84 { NULL }
6aa8b732
AK		 85};
86
87static struct dentry *debugfs_dir;
88
89#define MAX_IO_MSRS 256
90
707d92fa
RR		 91#define CR0_RESERVED_BITS \
92 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
93 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
94 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
66aee91a
RR		 95#define CR4_RESERVED_BITS \
96 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
97 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
98 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \
99 | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
100
7075bc81 101#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
6aa8b732
AK		 102#define EFER_RESERVED_BITS 0xfffffffffffff2fe
103
05b3e0c2 104#ifdef CONFIG_X86_64
6aa8b732
AK		 105// LDT or TSS descriptor in the GDT. 16 bytes.
106struct segment_descriptor_64 {
107 struct segment_descriptor s;
108 u32 base_higher;
109 u32 pad_zero;
110};
111
112#endif
113
bccf2150
AK		 114static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
115 unsigned long arg);
116
6aa8b732
AK		 117unsigned long segment_base(u16 selector)
118{
119 struct descriptor_table gdt;
120 struct segment_descriptor *d;
121 unsigned long table_base;
122 typedef unsigned long ul;
123 unsigned long v;
124
125 if (selector == 0)
126 return 0;
127
128 asm ("sgdt %0" : "=m"(gdt));
129 table_base = gdt.base;
130
131 if (selector & 4) { /* from ldt */
132 u16 ldt_selector;
133
134 asm ("sldt %0" : "=g"(ldt_selector));
135 table_base = segment_base(ldt_selector);
136 }
137 d = (struct segment_descriptor *)(table_base + (selector & ~7));
138 v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
05b3e0c2 139#ifdef CONFIG_X86_64
6aa8b732
AK		 140 if (d->system == 0
141 && (d->type == 2 || d->type == 9 || d->type == 11))
142 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
143#endif
144 return v;
145}
146EXPORT_SYMBOL_GPL(segment_base);
147
5aacf0ca
JM		 148static inline int valid_vcpu(int n)
149{
150 return likely(n >= 0 && n < KVM_MAX_VCPUS);
151}
152
7702fd1f
AK		 153void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
154{
155 if (!vcpu->fpu_active || vcpu->guest_fpu_loaded)
156 return;
157
158 vcpu->guest_fpu_loaded = 1;
b114b080
RR		 159 fx_save(&vcpu->host_fx_image);
160 fx_restore(&vcpu->guest_fx_image);
7702fd1f
AK		 161}
162EXPORT_SYMBOL_GPL(kvm_load_guest_fpu);
163
164void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
165{
166 if (!vcpu->guest_fpu_loaded)
167 return;
168
169 vcpu->guest_fpu_loaded = 0;
b114b080
RR		 170 fx_save(&vcpu->guest_fx_image);
171 fx_restore(&vcpu->host_fx_image);
7702fd1f
AK		 172}
173EXPORT_SYMBOL_GPL(kvm_put_guest_fpu);
174
bccf2150
AK		 175/*
176 * Switches to specified vcpu, until a matching vcpu_put()
177 */
178static void vcpu_load(struct kvm_vcpu *vcpu)
6aa8b732 179{
15ad7146
AK		 180 int cpu;
181
bccf2150 182 mutex_lock(&vcpu->mutex);
15ad7146
AK		 183 cpu = get_cpu();
184 preempt_notifier_register(&vcpu->preempt_notifier);
cbdd1bea 185 kvm_x86_ops->vcpu_load(vcpu, cpu);
15ad7146 186 put_cpu();
6aa8b732
AK		 187}
188
6aa8b732
AK		 189static void vcpu_put(struct kvm_vcpu *vcpu)
190{
15ad7146 191 preempt_disable();
cbdd1bea 192 kvm_x86_ops->vcpu_put(vcpu);
15ad7146
AK		 193 preempt_notifier_unregister(&vcpu->preempt_notifier);
194 preempt_enable();
6aa8b732
AK		 195 mutex_unlock(&vcpu->mutex);
196}
197
d9e368d6
AK		 198static void ack_flush(void *_completed)
199{
200 atomic_t *completed = _completed;
201
202 atomic_inc(completed);
203}
204
205void kvm_flush_remote_tlbs(struct kvm *kvm)
206{
207 int i, cpu, needed;
208 cpumask_t cpus;
209 struct kvm_vcpu *vcpu;
210 atomic_t completed;
211
212 atomic_set(&completed, 0);
213 cpus_clear(cpus);
214 needed = 0;
fb3f0f51
RR		 215 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
216 vcpu = kvm->vcpus[i];
217 if (!vcpu)
218 continue;
d9e368d6
AK		 219 if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests))
220 continue;
221 cpu = vcpu->cpu;
222 if (cpu != -1 && cpu != raw_smp_processor_id())
223 if (!cpu_isset(cpu, cpus)) {
224 cpu_set(cpu, cpus);
225 ++needed;
226 }
227 }
228
229 /*
230 * We really want smp_call_function_mask() here. But that's not
231 * available, so ipi all cpus in parallel and wait for them
232 * to complete.
233 */
234 for (cpu = first_cpu(cpus); cpu != NR_CPUS; cpu = next_cpu(cpu, cpus))
235 smp_call_function_single(cpu, ack_flush, &completed, 1, 0);
236 while (atomic_read(&completed) != needed) {
237 cpu_relax();
238 barrier();
239 }
240}
241
fb3f0f51
RR		 242int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
243{
244 struct page *page;
245 int r;
246
247 mutex_init(&vcpu->mutex);
248 vcpu->cpu = -1;
249 vcpu->mmu.root_hpa = INVALID_PAGE;
250 vcpu->kvm = kvm;
251 vcpu->vcpu_id = id;
c5ec1534
HQ		 252 if (!irqchip_in_kernel(kvm) || id == 0)
253 vcpu->mp_state = VCPU_MP_STATE_RUNNABLE;
254 else
255 vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED;
b6958ce4 256 init_waitqueue_head(&vcpu->wq);
fb3f0f51
RR		 257
258 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
259 if (!page) {
260 r = -ENOMEM;
261 goto fail;
262 }
263 vcpu->run = page_address(page);
264
265 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
266 if (!page) {
267 r = -ENOMEM;
268 goto fail_free_run;
269 }
270 vcpu->pio_data = page_address(page);
271
fb3f0f51
RR		 272 r = kvm_mmu_create(vcpu);
273 if (r < 0)
274 goto fail_free_pio_data;
275
276 return 0;
277
278fail_free_pio_data:
279 free_page((unsigned long)vcpu->pio_data);
280fail_free_run:
281 free_page((unsigned long)vcpu->run);
282fail:
283 return -ENOMEM;
284}
285EXPORT_SYMBOL_GPL(kvm_vcpu_init);
286
287void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
288{
289 kvm_mmu_destroy(vcpu);
1b9778da
ED		 290 if (vcpu->apic)
291 hrtimer_cancel(&vcpu->apic->timer.dev);
97222cc8 292 kvm_free_apic(vcpu->apic);
fb3f0f51
RR		 293 free_page((unsigned long)vcpu->pio_data);
294 free_page((unsigned long)vcpu->run);
295}
296EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
297
f17abe9a 298static struct kvm *kvm_create_vm(void)
6aa8b732
AK		 299{
300 struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
6aa8b732
AK		 301
302 if (!kvm)
f17abe9a 303 return ERR_PTR(-ENOMEM);
6aa8b732 304
74906345 305 kvm_io_bus_init(&kvm->pio_bus);
11ec2804 306 mutex_init(&kvm->lock);
6aa8b732 307 INIT_LIST_HEAD(&kvm->active_mmu_pages);
2eeb2e94 308 kvm_io_bus_init(&kvm->mmio_bus);
5e58cfe4
RR		 309 spin_lock(&kvm_lock);
310 list_add(&kvm->vm_list, &vm_list);
311 spin_unlock(&kvm_lock);
f17abe9a
AK		 312 return kvm;
313}
314
6aa8b732
AK		 315/*
316 * Free any memory in @free but not in @dont.
317 */
318static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
319 struct kvm_memory_slot *dont)
320{
321 int i;
322
323 if (!dont || free->phys_mem != dont->phys_mem)
324 if (free->phys_mem) {
325 for (i = 0; i < free->npages; ++i)
55a54f79
AK		 326 if (free->phys_mem[i])
327 __free_page(free->phys_mem[i]);
6aa8b732
AK		 328 vfree(free->phys_mem);
329 }
330
331 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
332 vfree(free->dirty_bitmap);
333
8b6d44c7 334 free->phys_mem = NULL;
6aa8b732 335 free->npages = 0;
8b6d44c7 336 free->dirty_bitmap = NULL;
6aa8b732
AK		 337}
338
339static void kvm_free_physmem(struct kvm *kvm)
340{
341 int i;
342
343 for (i = 0; i < kvm->nmemslots; ++i)
8b6d44c7 344 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
6aa8b732
AK		 345}
346
039576c0
AK		 347static void free_pio_guest_pages(struct kvm_vcpu *vcpu)
348{
349 int i;
350
3077c451 351 for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i)
039576c0
AK		 352 if (vcpu->pio.guest_pages[i]) {
353 __free_page(vcpu->pio.guest_pages[i]);
354 vcpu->pio.guest_pages[i] = NULL;
355 }
356}
357
7b53aa56
AK		 358static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
359{
7b53aa56
AK		 360 vcpu_load(vcpu);
361 kvm_mmu_unload(vcpu);
362 vcpu_put(vcpu);
363}
364
6aa8b732
AK		 365static void kvm_free_vcpus(struct kvm *kvm)
366{
367 unsigned int i;
368
7b53aa56
AK		 369 /*
370 * Unpin any mmu pages first.
371 */
372 for (i = 0; i < KVM_MAX_VCPUS; ++i)
fb3f0f51
RR		 373 if (kvm->vcpus[i])
374 kvm_unload_vcpu_mmu(kvm->vcpus[i]);
375 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
376 if (kvm->vcpus[i]) {
cbdd1bea 377 kvm_x86_ops->vcpu_free(kvm->vcpus[i]);
fb3f0f51
RR		 378 kvm->vcpus[i] = NULL;
379 }
380 }
381
6aa8b732
AK		 382}
383
f17abe9a
AK		 384static void kvm_destroy_vm(struct kvm *kvm)
385{
133de902
AK		 386 spin_lock(&kvm_lock);
387 list_del(&kvm->vm_list);
388 spin_unlock(&kvm_lock);
74906345 389 kvm_io_bus_destroy(&kvm->pio_bus);
2eeb2e94 390 kvm_io_bus_destroy(&kvm->mmio_bus);
85f455f7 391 kfree(kvm->vpic);
1fd4f2a5 392 kfree(kvm->vioapic);
6aa8b732
AK		 393 kvm_free_vcpus(kvm);
394 kvm_free_physmem(kvm);
395 kfree(kvm);
f17abe9a
AK		 396}
397
398static int kvm_vm_release(struct inode *inode, struct file *filp)
399{
400 struct kvm *kvm = filp->private_data;
401
402 kvm_destroy_vm(kvm);
6aa8b732
AK		 403 return 0;
404}
405
406static void inject_gp(struct kvm_vcpu *vcpu)
407{
cbdd1bea 408 kvm_x86_ops->inject_gp(vcpu, 0);
6aa8b732
AK		 409}
410
1342d353
AK		 411/*
412 * Load the pae pdptrs. Return true is they are all valid.
413 */
414static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
6aa8b732
AK		 415{
416 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
1342d353 417 unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
6aa8b732 418 int i;
6aa8b732 419 u64 *pdpt;
1342d353 420 int ret;
954bbbc2 421 struct page *page;
c820c2aa 422 u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)];
6aa8b732 423
11ec2804 424 mutex_lock(&vcpu->kvm->lock);
954bbbc2 425 page = gfn_to_page(vcpu->kvm, pdpt_gfn);
c820c2aa
RR		 426 if (!page) {
427 ret = 0;
428 goto out;
429 }
430
954bbbc2 431 pdpt = kmap_atomic(page, KM_USER0);
c820c2aa
RR		 432 memcpy(pdpte, pdpt+offset, sizeof(pdpte));
433 kunmap_atomic(pdpt, KM_USER0);
6aa8b732 434
c820c2aa
RR		 435 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
436 if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) {
1342d353
AK		 437 ret = 0;
438 goto out;
439 }
6aa8b732 440 }
c820c2aa 441 ret = 1;
6aa8b732 442
c820c2aa 443 memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs));
1342d353 444out:
11ec2804 445 mutex_unlock(&vcpu->kvm->lock);
6aa8b732 446
1342d353 447 return ret;
6aa8b732
AK		 448}
449
450void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
451{
707d92fa 452 if (cr0 & CR0_RESERVED_BITS) {
6aa8b732
AK		 453 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
454 cr0, vcpu->cr0);
455 inject_gp(vcpu);
456 return;
457 }
458
707d92fa 459 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) {
6aa8b732
AK		 460 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
461 inject_gp(vcpu);
462 return;
463 }
464
707d92fa 465 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) {
6aa8b732
AK		 466 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
467 "and a clear PE flag\n");
468 inject_gp(vcpu);
469 return;
470 }
471
707d92fa 472 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
05b3e0c2 473#ifdef CONFIG_X86_64
6aa8b732
AK		 474 if ((vcpu->shadow_efer & EFER_LME)) {
475 int cs_db, cs_l;
476
477 if (!is_pae(vcpu)) {
478 printk(KERN_DEBUG "set_cr0: #GP, start paging "
479 "in long mode while PAE is disabled\n");
480 inject_gp(vcpu);
481 return;
482 }
cbdd1bea 483 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
6aa8b732
AK		 484 if (cs_l) {
485 printk(KERN_DEBUG "set_cr0: #GP, start paging "
486 "in long mode while CS.L == 1\n");
487 inject_gp(vcpu);
488 return;
489
490 }
491 } else
492#endif
1342d353 493 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
6aa8b732
AK		 494 printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
495 "reserved bits\n");
496 inject_gp(vcpu);
497 return;
498 }
499
500 }
501
cbdd1bea 502 kvm_x86_ops->set_cr0(vcpu, cr0);
6aa8b732
AK		 503 vcpu->cr0 = cr0;
504
11ec2804 505 mutex_lock(&vcpu->kvm->lock);
6aa8b732 506 kvm_mmu_reset_context(vcpu);
11ec2804 507 mutex_unlock(&vcpu->kvm->lock);
6aa8b732
AK		 508 return;
509}
510EXPORT_SYMBOL_GPL(set_cr0);
511
512void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
513{
514 set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
515}
516EXPORT_SYMBOL_GPL(lmsw);
517
518void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
519{
66aee91a 520 if (cr4 & CR4_RESERVED_BITS) {
6aa8b732
AK		 521 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
522 inject_gp(vcpu);
523 return;
524 }
525
a9058ecd 526 if (is_long_mode(vcpu)) {
66aee91a 527 if (!(cr4 & X86_CR4_PAE)) {
6aa8b732
AK		 528 printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
529 "in long mode\n");
530 inject_gp(vcpu);
531 return;
532 }
66aee91a 533 } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE)
1342d353 534 && !load_pdptrs(vcpu, vcpu->cr3)) {
6aa8b732
AK		 535 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
536 inject_gp(vcpu);
310bc76c 537 return;
6aa8b732
AK		 538 }
539
66aee91a 540 if (cr4 & X86_CR4_VMXE) {
6aa8b732
AK		 541 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
542 inject_gp(vcpu);
543 return;
544 }
cbdd1bea 545 kvm_x86_ops->set_cr4(vcpu, cr4);
81f50e3b 546 vcpu->cr4 = cr4;
11ec2804 547 mutex_lock(&vcpu->kvm->lock);
6aa8b732 548 kvm_mmu_reset_context(vcpu);
11ec2804 549 mutex_unlock(&vcpu->kvm->lock);
6aa8b732
AK		 550}
551EXPORT_SYMBOL_GPL(set_cr4);
552
553void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
554{
a9058ecd 555 if (is_long_mode(vcpu)) {
f802a307 556 if (cr3 & CR3_L_MODE_RESERVED_BITS) {
6aa8b732
AK		 557 printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
558 inject_gp(vcpu);
559 return;
560 }
561 } else {
f802a307
RR		 562 if (is_pae(vcpu)) {
563 if (cr3 & CR3_PAE_RESERVED_BITS) {
564 printk(KERN_DEBUG
565 "set_cr3: #GP, reserved bits\n");
566 inject_gp(vcpu);
567 return;
568 }
569 if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) {
570 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
571 "reserved bits\n");
572 inject_gp(vcpu);
573 return;
574 }
575 } else {
576 if (cr3 & CR3_NONPAE_RESERVED_BITS) {
577 printk(KERN_DEBUG
578 "set_cr3: #GP, reserved bits\n");
579 inject_gp(vcpu);
580 return;
581 }
6aa8b732
AK		 582 }
583 }
584
11ec2804 585 mutex_lock(&vcpu->kvm->lock);
d21225ee
IM		 586 /*
587 * Does the new cr3 value map to physical memory? (Note, we
588 * catch an invalid cr3 even in real-mode, because it would
589 * cause trouble later on when we turn on paging anyway.)
590 *
591 * A real CPU would silently accept an invalid cr3 and would
592 * attempt to use it - with largely undefined (and often hard
593 * to debug) behavior on the guest side.
594 */
595 if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
596 inject_gp(vcpu);
fb764416
RR		 597 else {
598 vcpu->cr3 = cr3;
d21225ee 599 vcpu->mmu.new_cr3(vcpu);
fb764416 600 }
11ec2804 601 mutex_unlock(&vcpu->kvm->lock);
6aa8b732
AK		 602}
603EXPORT_SYMBOL_GPL(set_cr3);
604
605void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
606{
7075bc81 607 if (cr8 & CR8_RESERVED_BITS) {
6aa8b732
AK		 608 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
609 inject_gp(vcpu);
610 return;
611 }
97222cc8
ED		 612 if (irqchip_in_kernel(vcpu->kvm))
613 kvm_lapic_set_tpr(vcpu, cr8);
614 else
615 vcpu->cr8 = cr8;
6aa8b732
AK		 616}
617EXPORT_SYMBOL_GPL(set_cr8);
618
7017fc3d
ED		 619unsigned long get_cr8(struct kvm_vcpu *vcpu)
620{
97222cc8
ED		 621 if (irqchip_in_kernel(vcpu->kvm))
622 return kvm_lapic_get_cr8(vcpu);
623 else
624 return vcpu->cr8;
7017fc3d
ED		 625}
626EXPORT_SYMBOL_GPL(get_cr8);
627
628u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
629{
97222cc8
ED		 630 if (irqchip_in_kernel(vcpu->kvm))
631 return vcpu->apic_base;
632 else
633 return vcpu->apic_base;
7017fc3d
ED		 634}
635EXPORT_SYMBOL_GPL(kvm_get_apic_base);
636
637void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
638{
97222cc8
ED		 639 /* TODO: reserve bits check */
640 if (irqchip_in_kernel(vcpu->kvm))
641 kvm_lapic_set_base(vcpu, data);
642 else
643 vcpu->apic_base = data;
7017fc3d
ED		 644}
645EXPORT_SYMBOL_GPL(kvm_set_apic_base);
646
6aa8b732
AK		 647void fx_init(struct kvm_vcpu *vcpu)
648{
b114b080 649 unsigned after_mxcsr_mask;
6aa8b732 650
9bd01506
RR		 651 /* Initialize guest FPU by resetting ours and saving into guest's */
652 preempt_disable();
b114b080 653 fx_save(&vcpu->host_fx_image);
6aa8b732 654 fpu_init();
b114b080
RR		 655 fx_save(&vcpu->guest_fx_image);
656 fx_restore(&vcpu->host_fx_image);
9bd01506 657 preempt_enable();
6aa8b732 658
380102c8 659 vcpu->cr0 |= X86_CR0_ET;
b114b080
RR		 660 after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
661 vcpu->guest_fx_image.mxcsr = 0x1f80;
662 memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask,
663 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
6aa8b732
AK		 664}
665EXPORT_SYMBOL_GPL(fx_init);
666
6aa8b732
AK		 667/*
668 * Allocate some memory and give it an address in the guest physical address
669 * space.
670 *
671 * Discontiguous memory is allowed, mostly for framebuffers.
672 */
2c6f5df9
AK		 673static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
674 struct kvm_memory_region *mem)
6aa8b732
AK		 675{
676 int r;
677 gfn_t base_gfn;
678 unsigned long npages;
679 unsigned long i;
680 struct kvm_memory_slot *memslot;
681 struct kvm_memory_slot old, new;
6aa8b732
AK		 682
683 r = -EINVAL;
684 /* General sanity checks */
685 if (mem->memory_size & (PAGE_SIZE - 1))
686 goto out;
687 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
688 goto out;
689 if (mem->slot >= KVM_MEMORY_SLOTS)
690 goto out;
691 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
692 goto out;
693
694 memslot = &kvm->memslots[mem->slot];
695 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
696 npages = mem->memory_size >> PAGE_SHIFT;
697
698 if (!npages)
699 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
700
11ec2804 701 mutex_lock(&kvm->lock);
6aa8b732 702
6aa8b732
AK		 703 new = old = *memslot;
704
705 new.base_gfn = base_gfn;
706 new.npages = npages;
707 new.flags = mem->flags;
708
709 /* Disallow changing a memory slot's size. */
710 r = -EINVAL;
711 if (npages && old.npages && npages != old.npages)
712 goto out_unlock;
713
714 /* Check for overlaps */
715 r = -EEXIST;
716 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
717 struct kvm_memory_slot *s = &kvm->memslots[i];
718
719 if (s == memslot)
720 continue;
721 if (!((base_gfn + npages <= s->base_gfn) ||
722 (base_gfn >= s->base_gfn + s->npages)))
723 goto out_unlock;
724 }
6aa8b732
AK		 725
726 /* Deallocate if slot is being removed */
727 if (!npages)
8b6d44c7 728 new.phys_mem = NULL;
6aa8b732
AK		 729
730 /* Free page dirty bitmap if unneeded */
731 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
8b6d44c7 732 new.dirty_bitmap = NULL;
6aa8b732
AK		 733
734 r = -ENOMEM;
735
736 /* Allocate if a slot is being created */
737 if (npages && !new.phys_mem) {
738 new.phys_mem = vmalloc(npages * sizeof(struct page *));
739
740 if (!new.phys_mem)
0d8d2bd4 741 goto out_unlock;
6aa8b732
AK		 742
743 memset(new.phys_mem, 0, npages * sizeof(struct page *));
744 for (i = 0; i < npages; ++i) {
745 new.phys_mem[i] = alloc_page(GFP_HIGHUSER
746 | __GFP_ZERO);
747 if (!new.phys_mem[i])
0d8d2bd4 748 goto out_unlock;
5972e953 749 set_page_private(new.phys_mem[i],0);
6aa8b732
AK		 750 }
751 }
752
753 /* Allocate page dirty bitmap if needed */
754 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
755 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
756
757 new.dirty_bitmap = vmalloc(dirty_bytes);
758 if (!new.dirty_bitmap)
0d8d2bd4 759 goto out_unlock;
6aa8b732
AK		 760 memset(new.dirty_bitmap, 0, dirty_bytes);
761 }
762
6aa8b732
AK		 763 if (mem->slot >= kvm->nmemslots)
764 kvm->nmemslots = mem->slot + 1;
765
766 *memslot = new;
6aa8b732 767
90cb0529
AK		 768 kvm_mmu_slot_remove_write_access(kvm, mem->slot);
769 kvm_flush_remote_tlbs(kvm);
6aa8b732 770
11ec2804 771 mutex_unlock(&kvm->lock);
6aa8b732
AK		 772
773 kvm_free_physmem_slot(&old, &new);
774 return 0;
775
776out_unlock:
11ec2804 777 mutex_unlock(&kvm->lock);
6aa8b732
AK		 778 kvm_free_physmem_slot(&new, &old);
779out:
780 return r;
781}
782
783/*
784 * Get (and clear) the dirty memory log for a memory slot.
785 */
2c6f5df9
AK		 786static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
787 struct kvm_dirty_log *log)
6aa8b732
AK		 788{
789 struct kvm_memory_slot *memslot;
790 int r, i;
791 int n;
792 unsigned long any = 0;
793
11ec2804 794 mutex_lock(&kvm->lock);
6aa8b732 795
6aa8b732
AK		 796 r = -EINVAL;
797 if (log->slot >= KVM_MEMORY_SLOTS)
798 goto out;
799
800 memslot = &kvm->memslots[log->slot];
801 r = -ENOENT;
802 if (!memslot->dirty_bitmap)
803 goto out;
804
cd1a4a98 805 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
6aa8b732 806
cd1a4a98 807 for (i = 0; !any && i < n/sizeof(long); ++i)
6aa8b732
AK		 808 any = memslot->dirty_bitmap[i];
809
810 r = -EFAULT;
811 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
812 goto out;
813
39214915
RR		 814 /* If nothing is dirty, don't bother messing with page tables. */
815 if (any) {
39214915
RR		 816 kvm_mmu_slot_remove_write_access(kvm, log->slot);
817 kvm_flush_remote_tlbs(kvm);
818 memset(memslot->dirty_bitmap, 0, n);
39214915 819 }
6aa8b732
AK		 820
821 r = 0;
822
823out:
11ec2804 824 mutex_unlock(&kvm->lock);
6aa8b732
AK		 825 return r;
826}
827
e8207547
AK		 828/*
829 * Set a new alias region. Aliases map a portion of physical memory into
830 * another portion. This is useful for memory windows, for example the PC
831 * VGA region.
832 */
833static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
834 struct kvm_memory_alias *alias)
835{
836 int r, n;
837 struct kvm_mem_alias *p;
838
839 r = -EINVAL;
840 /* General sanity checks */
841 if (alias->memory_size & (PAGE_SIZE - 1))
842 goto out;
843 if (alias->guest_phys_addr & (PAGE_SIZE - 1))
844 goto out;
845 if (alias->slot >= KVM_ALIAS_SLOTS)
846 goto out;
847 if (alias->guest_phys_addr + alias->memory_size
848 < alias->guest_phys_addr)
849 goto out;
850 if (alias->target_phys_addr + alias->memory_size
851 < alias->target_phys_addr)
852 goto out;
853
11ec2804 854 mutex_lock(&kvm->lock);
e8207547
AK		 855
856 p = &kvm->aliases[alias->slot];
857 p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
858 p->npages = alias->memory_size >> PAGE_SHIFT;
859 p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
860
861 for (n = KVM_ALIAS_SLOTS; n > 0; --n)
862 if (kvm->aliases[n - 1].npages)
863 break;
864 kvm->naliases = n;
865
90cb0529 866 kvm_mmu_zap_all(kvm);
e8207547 867
11ec2804 868 mutex_unlock(&kvm->lock);
e8207547
AK		 869
870 return 0;
871
872out:
873 return r;
874}
875
6ceb9d79
HQ		 876static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
877{
878 int r;
879
880 r = 0;
881 switch (chip->chip_id) {
882 case KVM_IRQCHIP_PIC_MASTER:
883 memcpy (&chip->chip.pic,
884 &pic_irqchip(kvm)->pics[0],
885 sizeof(struct kvm_pic_state));
886 break;
887 case KVM_IRQCHIP_PIC_SLAVE:
888 memcpy (&chip->chip.pic,
889 &pic_irqchip(kvm)->pics[1],
890 sizeof(struct kvm_pic_state));
891 break;
6bf9e962
HQ		 892 case KVM_IRQCHIP_IOAPIC:
893 memcpy (&chip->chip.ioapic,
894 ioapic_irqchip(kvm),
895 sizeof(struct kvm_ioapic_state));
896 break;
6ceb9d79
HQ		 897 default:
898 r = -EINVAL;
899 break;
900 }
901 return r;
902}
903
904static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
905{
906 int r;
907
908 r = 0;
909 switch (chip->chip_id) {
910 case KVM_IRQCHIP_PIC_MASTER:
911 memcpy (&pic_irqchip(kvm)->pics[0],
912 &chip->chip.pic,
913 sizeof(struct kvm_pic_state));
914 break;
915 case KVM_IRQCHIP_PIC_SLAVE:
916 memcpy (&pic_irqchip(kvm)->pics[1],
917 &chip->chip.pic,
918 sizeof(struct kvm_pic_state));
919 break;
6bf9e962
HQ		 920 case KVM_IRQCHIP_IOAPIC:
921 memcpy (ioapic_irqchip(kvm),
922 &chip->chip.ioapic,
923 sizeof(struct kvm_ioapic_state));
924 break;
6ceb9d79
HQ		 925 default:
926 r = -EINVAL;
927 break;
928 }
929 kvm_pic_update_irq(pic_irqchip(kvm));
930 return r;
931}
932
e8207547
AK		 933static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
934{
935 int i;
936 struct kvm_mem_alias *alias;
937
938 for (i = 0; i < kvm->naliases; ++i) {
939 alias = &kvm->aliases[i];
940 if (gfn >= alias->base_gfn
941 && gfn < alias->base_gfn + alias->npages)
942 return alias->target_gfn + gfn - alias->base_gfn;
943 }
944 return gfn;
945}
946
947static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
6aa8b732
AK		 948{
949 int i;
950
951 for (i = 0; i < kvm->nmemslots; ++i) {
952 struct kvm_memory_slot *memslot = &kvm->memslots[i];
953
954 if (gfn >= memslot->base_gfn
955 && gfn < memslot->base_gfn + memslot->npages)
956 return memslot;
957 }
8b6d44c7 958 return NULL;
6aa8b732 959}
e8207547
AK		 960
961struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
962{
963 gfn = unalias_gfn(kvm, gfn);
964 return __gfn_to_memslot(kvm, gfn);
965}
6aa8b732 966
954bbbc2
AK		 967struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
968{
969 struct kvm_memory_slot *slot;
970
e8207547
AK		 971 gfn = unalias_gfn(kvm, gfn);
972 slot = __gfn_to_memslot(kvm, gfn);
954bbbc2
AK		 973 if (!slot)
974 return NULL;
975 return slot->phys_mem[gfn - slot->base_gfn];
976}
977EXPORT_SYMBOL_GPL(gfn_to_page);
978
7e9d619d 979/* WARNING: Does not work on aliased pages. */
6aa8b732
AK		 980void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
981{
31389947 982 struct kvm_memory_slot *memslot;
6aa8b732 983
7e9d619d
RR		 984 memslot = __gfn_to_memslot(kvm, gfn);
985 if (memslot && memslot->dirty_bitmap) {
986 unsigned long rel_gfn = gfn - memslot->base_gfn;
6aa8b732 987
7e9d619d
RR		 988 /* avoid RMW */
989 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
990 set_bit(rel_gfn, memslot->dirty_bitmap);
6aa8b732
AK		 991 }
992}
993
e7d5d76c 994int emulator_read_std(unsigned long addr,
4c690a1e 995 void *val,
6aa8b732 996 unsigned int bytes,
cebff02b 997 struct kvm_vcpu *vcpu)
6aa8b732 998{
6aa8b732
AK		 999 void *data = val;
1000
1001 while (bytes) {
1002 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1003 unsigned offset = addr & (PAGE_SIZE-1);
1004 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
1005 unsigned long pfn;
954bbbc2
AK		 1006 struct page *page;
1007 void *page_virt;
6aa8b732
AK		 1008
1009 if (gpa == UNMAPPED_GVA)
1010 return X86EMUL_PROPAGATE_FAULT;
1011 pfn = gpa >> PAGE_SHIFT;
954bbbc2
AK		 1012 page = gfn_to_page(vcpu->kvm, pfn);
1013 if (!page)
6aa8b732 1014 return X86EMUL_UNHANDLEABLE;
954bbbc2 1015 page_virt = kmap_atomic(page, KM_USER0);
6aa8b732 1016
954bbbc2 1017 memcpy(data, page_virt + offset, tocopy);
6aa8b732 1018
954bbbc2 1019 kunmap_atomic(page_virt, KM_USER0);
6aa8b732
AK		 1020
1021 bytes -= tocopy;
1022 data += tocopy;
1023 addr += tocopy;
1024 }
1025
1026 return X86EMUL_CONTINUE;
1027}
e7d5d76c 1028EXPORT_SYMBOL_GPL(emulator_read_std);
6aa8b732
AK		 1029
1030static int emulator_write_std(unsigned long addr,
4c690a1e 1031 const void *val,
6aa8b732 1032 unsigned int bytes,
cebff02b 1033 struct kvm_vcpu *vcpu)
6aa8b732 1034{
f0242478 1035 pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes);
6aa8b732
AK		 1036 return X86EMUL_UNHANDLEABLE;
1037}
1038
97222cc8
ED		 1039/*
1040 * Only apic need an MMIO device hook, so shortcut now..
1041 */
1042static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu,
1043 gpa_t addr)
1044{
1045 struct kvm_io_device *dev;
1046
1047 if (vcpu->apic) {
1048 dev = &vcpu->apic->dev;
1049 if (dev->in_range(dev, addr))
1050 return dev;
1051 }
1052 return NULL;
1053}
1054
2eeb2e94
GH		 1055static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
1056 gpa_t addr)
1057{
97222cc8
ED		 1058 struct kvm_io_device *dev;
1059
1060 dev = vcpu_find_pervcpu_dev(vcpu, addr);
1061 if (dev == NULL)
1062 dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr);
1063 return dev;
2eeb2e94
GH		 1064}
1065
74906345
ED		 1066static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu,
1067 gpa_t addr)
1068{
1069 return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr);
1070}
1071
6aa8b732 1072static int emulator_read_emulated(unsigned long addr,
4c690a1e 1073 void *val,
6aa8b732 1074 unsigned int bytes,
cebff02b 1075 struct kvm_vcpu *vcpu)
6aa8b732 1076{
2eeb2e94
GH		 1077 struct kvm_io_device *mmio_dev;
1078 gpa_t gpa;
6aa8b732
AK		 1079
1080 if (vcpu->mmio_read_completed) {
1081 memcpy(val, vcpu->mmio_data, bytes);
1082 vcpu->mmio_read_completed = 0;
1083 return X86EMUL_CONTINUE;
cebff02b 1084 } else if (emulator_read_std(addr, val, bytes, vcpu)
6aa8b732
AK		 1085 == X86EMUL_CONTINUE)
1086 return X86EMUL_CONTINUE;
d27d4aca 1087
2eeb2e94
GH		 1088 gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
1089 if (gpa == UNMAPPED_GVA)
1090 return X86EMUL_PROPAGATE_FAULT;
6aa8b732 1091
2eeb2e94
GH		 1092 /*
1093 * Is this MMIO handled locally?
1094 */
1095 mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1096 if (mmio_dev) {
1097 kvm_iodevice_read(mmio_dev, gpa, bytes, val);
1098 return X86EMUL_CONTINUE;
6aa8b732 1099 }
2eeb2e94
GH		 1100
1101 vcpu->mmio_needed = 1;
1102 vcpu->mmio_phys_addr = gpa;
1103 vcpu->mmio_size = bytes;
1104 vcpu->mmio_is_write = 0;
1105
1106 return X86EMUL_UNHANDLEABLE;
6aa8b732
AK		 1107}
1108
da4a00f0 1109static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
4c690a1e 1110 const void *val, int bytes)
da4a00f0 1111{
da4a00f0
AK		 1112 struct page *page;
1113 void *virt;
1114
1115 if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
1116 return 0;
954bbbc2
AK		 1117 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1118 if (!page)
da4a00f0 1119 return 0;
ab51a434 1120 mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT);
da4a00f0 1121 virt = kmap_atomic(page, KM_USER0);
fe551881 1122 kvm_mmu_pte_write(vcpu, gpa, val, bytes);
7cfa4b0a 1123 memcpy(virt + offset_in_page(gpa), val, bytes);
da4a00f0 1124 kunmap_atomic(virt, KM_USER0);
da4a00f0
AK		 1125 return 1;
1126}
1127
b0fcd903
AK		 1128static int emulator_write_emulated_onepage(unsigned long addr,
1129 const void *val,
1130 unsigned int bytes,
cebff02b 1131 struct kvm_vcpu *vcpu)
6aa8b732 1132{
2eeb2e94
GH		 1133 struct kvm_io_device *mmio_dev;
1134 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
6aa8b732 1135
c9047f53 1136 if (gpa == UNMAPPED_GVA) {
cbdd1bea 1137 kvm_x86_ops->inject_page_fault(vcpu, addr, 2);
6aa8b732 1138 return X86EMUL_PROPAGATE_FAULT;
c9047f53 1139 }
6aa8b732 1140
da4a00f0
AK		 1141 if (emulator_write_phys(vcpu, gpa, val, bytes))
1142 return X86EMUL_CONTINUE;
1143
2eeb2e94
GH		 1144 /*
1145 * Is this MMIO handled locally?
1146 */
1147 mmio_dev = vcpu_find_mmio_dev(vcpu, gpa);
1148 if (mmio_dev) {
1149 kvm_iodevice_write(mmio_dev, gpa, bytes, val);
1150 return X86EMUL_CONTINUE;
1151 }
1152
6aa8b732
AK		 1153 vcpu->mmio_needed = 1;
1154 vcpu->mmio_phys_addr = gpa;
1155 vcpu->mmio_size = bytes;
1156 vcpu->mmio_is_write = 1;
4c690a1e 1157 memcpy(vcpu->mmio_data, val, bytes);
6aa8b732
AK		 1158
1159 return X86EMUL_CONTINUE;
1160}
1161
e7d5d76c 1162int emulator_write_emulated(unsigned long addr,
b0fcd903
AK		 1163 const void *val,
1164 unsigned int bytes,
cebff02b 1165 struct kvm_vcpu *vcpu)
b0fcd903
AK		 1166{
1167 /* Crossing a page boundary? */
1168 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
1169 int rc, now;
1170
1171 now = -addr & ~PAGE_MASK;
cebff02b 1172 rc = emulator_write_emulated_onepage(addr, val, now, vcpu);
b0fcd903
AK		 1173 if (rc != X86EMUL_CONTINUE)
1174 return rc;
1175 addr += now;
1176 val += now;
1177 bytes -= now;
1178 }
cebff02b 1179 return emulator_write_emulated_onepage(addr, val, bytes, vcpu);
b0fcd903 1180}
e7d5d76c 1181EXPORT_SYMBOL_GPL(emulator_write_emulated);
b0fcd903 1182
6aa8b732 1183static int emulator_cmpxchg_emulated(unsigned long addr,
4c690a1e
AK		 1184 const void *old,
1185 const void *new,
6aa8b732 1186 unsigned int bytes,
cebff02b 1187 struct kvm_vcpu *vcpu)
6aa8b732
AK		 1188{
1189 static int reported;
1190
1191 if (!reported) {
1192 reported = 1;
1193 printk(KERN_WARNING "kvm: emulating exchange as write\n");
1194 }
cebff02b 1195 return emulator_write_emulated(addr, new, bytes, vcpu);
6aa8b732
AK		 1196}
1197
1198static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
1199{
cbdd1bea 1200 return kvm_x86_ops->get_segment_base(vcpu, seg);
6aa8b732
AK		 1201}
1202
1203int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
1204{
6aa8b732
AK		 1205 return X86EMUL_CONTINUE;
1206}
1207
1208int emulate_clts(struct kvm_vcpu *vcpu)
1209{
81f50e3b 1210 vcpu->cr0 &= ~X86_CR0_TS;
cbdd1bea 1211 kvm_x86_ops->set_cr0(vcpu, vcpu->cr0);
6aa8b732
AK		 1212 return X86EMUL_CONTINUE;
1213}
1214
1215int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1216{
1217 struct kvm_vcpu *vcpu = ctxt->vcpu;
1218
1219 switch (dr) {
1220 case 0 ... 3:
cbdd1bea 1221 *dest = kvm_x86_ops->get_dr(vcpu, dr);
6aa8b732
AK		 1222 return X86EMUL_CONTINUE;
1223 default:
f0242478 1224 pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr);
6aa8b732
AK		 1225 return X86EMUL_UNHANDLEABLE;
1226 }
1227}
1228
1229int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1230{
1231 unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1232 int exception;
1233
cbdd1bea 1234 kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
6aa8b732
AK		 1235 if (exception) {
1236 /* FIXME: better handling */
1237 return X86EMUL_UNHANDLEABLE;
1238 }
1239 return X86EMUL_CONTINUE;
1240}
1241
1242static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1243{
1244 static int reported;
1245 u8 opcodes[4];
1246 unsigned long rip = ctxt->vcpu->rip;
1247 unsigned long rip_linear;
1248
1249 rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1250
1251 if (reported)
1252 return;
1253
cebff02b 1254 emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt->vcpu);
6aa8b732
AK		 1255
1256 printk(KERN_ERR "emulation failed but !mmio_needed?"
1257 " rip %lx %02x %02x %02x %02x\n",
1258 rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1259 reported = 1;
1260}
1261
1262struct x86_emulate_ops emulate_ops = {
1263 .read_std = emulator_read_std,
1264 .write_std = emulator_write_std,
1265 .read_emulated = emulator_read_emulated,
1266 .write_emulated = emulator_write_emulated,
1267 .cmpxchg_emulated = emulator_cmpxchg_emulated,
1268};
1269
1270int emulate_instruction(struct kvm_vcpu *vcpu,
1271 struct kvm_run *run,
1272 unsigned long cr2,
1273 u16 error_code)
1274{
1275 struct x86_emulate_ctxt emulate_ctxt;
1276 int r;
1277 int cs_db, cs_l;
1278
e7df56e4 1279 vcpu->mmio_fault_cr2 = cr2;
cbdd1bea 1280 kvm_x86_ops->cache_regs(vcpu);
6aa8b732 1281
cbdd1bea 1282 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
6aa8b732
AK		 1283
1284 emulate_ctxt.vcpu = vcpu;
cbdd1bea 1285 emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
6aa8b732
AK		 1286 emulate_ctxt.cr2 = cr2;
1287 emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1288 ? X86EMUL_MODE_REAL : cs_l
1289 ? X86EMUL_MODE_PROT64 : cs_db
1290 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1291
1292 if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1293 emulate_ctxt.cs_base = 0;
1294 emulate_ctxt.ds_base = 0;
1295 emulate_ctxt.es_base = 0;
1296 emulate_ctxt.ss_base = 0;
1297 } else {
1298 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1299 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1300 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1301 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1302 }
1303
1304 emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1305 emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1306
1307 vcpu->mmio_is_write = 0;
e70669ab 1308 vcpu->pio.string = 0;
6aa8b732 1309 r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
e70669ab
LV		 1310 if (vcpu->pio.string)
1311 return EMULATE_DO_MMIO;
6aa8b732
AK		 1312
1313 if ((r || vcpu->mmio_is_write) && run) {
8fc0d085 1314 run->exit_reason = KVM_EXIT_MMIO;
6aa8b732
AK		 1315 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1316 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1317 run->mmio.len = vcpu->mmio_size;
1318 run->mmio.is_write = vcpu->mmio_is_write;
1319 }
1320
1321 if (r) {
a436036b
AK		 1322 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1323 return EMULATE_DONE;
6aa8b732
AK		 1324 if (!vcpu->mmio_needed) {
1325 report_emulation_failure(&emulate_ctxt);
1326 return EMULATE_FAIL;
1327 }
1328 return EMULATE_DO_MMIO;
1329 }
1330
cbdd1bea
CE		 1331 kvm_x86_ops->decache_regs(vcpu);
1332 kvm_x86_ops->set_rflags(vcpu, emulate_ctxt.eflags);
6aa8b732 1333
02c83209
AK		 1334 if (vcpu->mmio_is_write) {
1335 vcpu->mmio_needed = 0;
6aa8b732 1336 return EMULATE_DO_MMIO;
02c83209 1337 }
6aa8b732
AK		 1338
1339 return EMULATE_DONE;
1340}
1341EXPORT_SYMBOL_GPL(emulate_instruction);
1342
b6958ce4
ED		 1343/*
1344 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1345 */
c5ec1534 1346static void kvm_vcpu_block(struct kvm_vcpu *vcpu)
d3bef15f 1347{
b6958ce4
ED		 1348 DECLARE_WAITQUEUE(wait, current);
1349
1350 add_wait_queue(&vcpu->wq, &wait);
1351
1352 /*
1353 * We will block until either an interrupt or a signal wakes us up
1354 */
c5ec1534
HQ		 1355 while (!kvm_cpu_has_interrupt(vcpu)
1356 && !signal_pending(current)
1357 && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE
1358 && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) {
b6958ce4
ED		 1359 set_current_state(TASK_INTERRUPTIBLE);
1360 vcpu_put(vcpu);
1361 schedule();
1362 vcpu_load(vcpu);
1363 }
d3bef15f 1364
c5ec1534 1365 __set_current_state(TASK_RUNNING);
b6958ce4 1366 remove_wait_queue(&vcpu->wq, &wait);
b6958ce4
ED		 1367}
1368
1369int kvm_emulate_halt(struct kvm_vcpu *vcpu)
1370{
d3bef15f 1371 ++vcpu->stat.halt_exits;
b6958ce4 1372 if (irqchip_in_kernel(vcpu->kvm)) {
c5ec1534
HQ		 1373 vcpu->mp_state = VCPU_MP_STATE_HALTED;
1374 kvm_vcpu_block(vcpu);
1375 if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE)
1376 return -EINTR;
b6958ce4
ED		 1377 return 1;
1378 } else {
1379 vcpu->run->exit_reason = KVM_EXIT_HLT;
1380 return 0;
1381 }
d3bef15f
AK		 1382}
1383EXPORT_SYMBOL_GPL(kvm_emulate_halt);
1384
270fd9b9
AK		 1385int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
1386{
1387 unsigned long nr, a0, a1, a2, a3, a4, a5, ret;
1388
cbdd1bea 1389 kvm_x86_ops->cache_regs(vcpu);
270fd9b9
AK		 1390 ret = -KVM_EINVAL;
1391#ifdef CONFIG_X86_64
1392 if (is_long_mode(vcpu)) {
1393 nr = vcpu->regs[VCPU_REGS_RAX];
1394 a0 = vcpu->regs[VCPU_REGS_RDI];
1395 a1 = vcpu->regs[VCPU_REGS_RSI];
1396 a2 = vcpu->regs[VCPU_REGS_RDX];
1397 a3 = vcpu->regs[VCPU_REGS_RCX];
1398 a4 = vcpu->regs[VCPU_REGS_R8];
1399 a5 = vcpu->regs[VCPU_REGS_R9];
1400 } else
1401#endif
1402 {
1403 nr = vcpu->regs[VCPU_REGS_RBX] & -1u;
1404 a0 = vcpu->regs[VCPU_REGS_RAX] & -1u;
1405 a1 = vcpu->regs[VCPU_REGS_RCX] & -1u;
1406 a2 = vcpu->regs[VCPU_REGS_RDX] & -1u;
1407 a3 = vcpu->regs[VCPU_REGS_RSI] & -1u;
1408 a4 = vcpu->regs[VCPU_REGS_RDI] & -1u;
1409 a5 = vcpu->regs[VCPU_REGS_RBP] & -1u;
1410 }
1411 switch (nr) {
1412 default:
519ef353 1413 run->hypercall.nr = nr;
b4e63f56
AK		 1414 run->hypercall.args[0] = a0;
1415 run->hypercall.args[1] = a1;
1416 run->hypercall.args[2] = a2;
1417 run->hypercall.args[3] = a3;
1418 run->hypercall.args[4] = a4;
1419 run->hypercall.args[5] = a5;
1420 run->hypercall.ret = ret;
1421 run->hypercall.longmode = is_long_mode(vcpu);
cbdd1bea 1422 kvm_x86_ops->decache_regs(vcpu);
b4e63f56 1423 return 0;
270fd9b9
AK		 1424 }
1425 vcpu->regs[VCPU_REGS_RAX] = ret;
cbdd1bea 1426 kvm_x86_ops->decache_regs(vcpu);
270fd9b9
AK		 1427 return 1;
1428}
1429EXPORT_SYMBOL_GPL(kvm_hypercall);
1430
6aa8b732
AK		 1431static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1432{
1433 return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1434}
1435
1436void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1437{
1438 struct descriptor_table dt = { limit, base };
1439
cbdd1bea 1440 kvm_x86_ops->set_gdt(vcpu, &dt);
6aa8b732
AK		 1441}
1442
1443void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1444{
1445 struct descriptor_table dt = { limit, base };
1446
cbdd1bea 1447 kvm_x86_ops->set_idt(vcpu, &dt);
6aa8b732
AK		 1448}
1449
1450void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1451 unsigned long *rflags)
1452{
1453 lmsw(vcpu, msw);
cbdd1bea 1454 *rflags = kvm_x86_ops->get_rflags(vcpu);
6aa8b732
AK		 1455}
1456
1457unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1458{
cbdd1bea 1459 kvm_x86_ops->decache_cr4_guest_bits(vcpu);
6aa8b732
AK		 1460 switch (cr) {
1461 case 0:
1462 return vcpu->cr0;
1463 case 2:
1464 return vcpu->cr2;
1465 case 3:
1466 return vcpu->cr3;
1467 case 4:
1468 return vcpu->cr4;
1469 default:
1470 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1471 return 0;
1472 }
1473}
1474
1475void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1476 unsigned long *rflags)
1477{
1478 switch (cr) {
1479 case 0:
1480 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
cbdd1bea 1481 *rflags = kvm_x86_ops->get_rflags(vcpu);
6aa8b732
AK		 1482 break;
1483 case 2:
1484 vcpu->cr2 = val;
1485 break;
1486 case 3:
1487 set_cr3(vcpu, val);
1488 break;
1489 case 4:
1490 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1491 break;
1492 default:
1493 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1494 }
1495}
1496
102d8325
IM		 1497/*
1498 * Register the para guest with the host:
1499 */
1500static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa)
1501{
1502 struct kvm_vcpu_para_state *para_state;
1503 hpa_t para_state_hpa, hypercall_hpa;
1504 struct page *para_state_page;
1505 unsigned char *hypercall;
1506 gpa_t hypercall_gpa;
1507
1508 printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n");
1509 printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa);
1510
1511 /*
1512 * Needs to be page aligned:
1513 */
1514 if (para_state_gpa != PAGE_ALIGN(para_state_gpa))
1515 goto err_gp;
1516
1517 para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa);
1518 printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa);
1519 if (is_error_hpa(para_state_hpa))
1520 goto err_gp;
1521
ab51a434 1522 mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT);
102d8325 1523 para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT);
fe551881 1524 para_state = kmap(para_state_page);
102d8325
IM		 1525
1526 printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version);
1527 printk(KERN_DEBUG ".... size: %d\n", para_state->size);
1528
1529 para_state->host_version = KVM_PARA_API_VERSION;
1530 /*
1531 * We cannot support guests that try to register themselves
1532 * with a newer API version than the host supports:
1533 */
1534 if (para_state->guest_version > KVM_PARA_API_VERSION) {
1535 para_state->ret = -KVM_EINVAL;
1536 goto err_kunmap_skip;
1537 }
1538
1539 hypercall_gpa = para_state->hypercall_gpa;
1540 hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa);
1541 printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa);
1542 if (is_error_hpa(hypercall_hpa)) {
1543 para_state->ret = -KVM_EINVAL;
1544 goto err_kunmap_skip;
1545 }
1546
1547 printk(KERN_DEBUG "kvm: para guest successfully registered.\n");
1548 vcpu->para_state_page = para_state_page;
1549 vcpu->para_state_gpa = para_state_gpa;
1550 vcpu->hypercall_gpa = hypercall_gpa;
1551
ab51a434 1552 mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT);
102d8325
IM		 1553 hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT),
1554 KM_USER1) + (hypercall_hpa & ~PAGE_MASK);
cbdd1bea 1555 kvm_x86_ops->patch_hypercall(vcpu, hypercall);
102d8325
IM		 1556 kunmap_atomic(hypercall, KM_USER1);
1557
1558 para_state->ret = 0;
1559err_kunmap_skip:
fe551881 1560 kunmap(para_state_page);
102d8325
IM		 1561 return 0;
1562err_gp:
1563 return 1;
1564}
1565
3bab1f5d
AK		 1566int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1567{
1568 u64 data;
1569
1570 switch (msr) {
1571 case 0xc0010010: /* SYSCFG */
1572 case 0xc0010015: /* HWCR */
1573 case MSR_IA32_PLATFORM_ID:
1574 case MSR_IA32_P5_MC_ADDR:
1575 case MSR_IA32_P5_MC_TYPE:
1576 case MSR_IA32_MC0_CTL:
1577 case MSR_IA32_MCG_STATUS:
1578 case MSR_IA32_MCG_CAP:
1579 case MSR_IA32_MC0_MISC:
1580 case MSR_IA32_MC0_MISC+4:
1581 case MSR_IA32_MC0_MISC+8:
1582 case MSR_IA32_MC0_MISC+12:
1583 case MSR_IA32_MC0_MISC+16:
1584 case MSR_IA32_UCODE_REV:
a8d13ea2 1585 case MSR_IA32_PERF_STATUS:
2dc7094b 1586 case MSR_IA32_EBL_CR_POWERON:
3bab1f5d
AK		 1587 /* MTRR registers */
1588 case 0xfe:
1589 case 0x200 ... 0x2ff:
1590 data = 0;
1591 break;
a8d13ea2
AK		 1592 case 0xcd: /* fsb frequency */
1593 data = 3;
1594 break;
3bab1f5d 1595 case MSR_IA32_APICBASE:
7017fc3d 1596 data = kvm_get_apic_base(vcpu);
3bab1f5d 1597 break;
6f00e68f
AK		 1598 case MSR_IA32_MISC_ENABLE:
1599 data = vcpu->ia32_misc_enable_msr;
1600 break;
3bab1f5d
AK		 1601#ifdef CONFIG_X86_64
1602 case MSR_EFER:
1603 data = vcpu->shadow_efer;
1604 break;
1605#endif
1606 default:
f0242478 1607 pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
3bab1f5d
AK		 1608 return 1;
1609 }
1610 *pdata = data;
1611 return 0;
1612}
1613EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1614
6aa8b732
AK		 1615/*
1616 * Reads an msr value (of 'msr_index') into 'pdata'.
1617 * Returns 0 on success, non-0 otherwise.
1618 * Assumes vcpu_load() was already called.
1619 */
35f3f286 1620int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
6aa8b732 1621{
cbdd1bea 1622 return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
6aa8b732
AK		 1623}
1624
05b3e0c2 1625#ifdef CONFIG_X86_64
6aa8b732 1626
3bab1f5d 1627static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
6aa8b732 1628{
6aa8b732
AK		 1629 if (efer & EFER_RESERVED_BITS) {
1630 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1631 efer);
1632 inject_gp(vcpu);
1633 return;
1634 }
1635
1636 if (is_paging(vcpu)
1637 && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1638 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1639 inject_gp(vcpu);
1640 return;
1641 }
1642
cbdd1bea 1643 kvm_x86_ops->set_efer(vcpu, efer);
7725f0ba 1644
6aa8b732
AK		 1645 efer &= ~EFER_LMA;
1646 efer |= vcpu->shadow_efer & EFER_LMA;
1647
1648 vcpu->shadow_efer = efer;
6aa8b732 1649}
6aa8b732
AK		 1650
1651#endif
1652
3bab1f5d
AK		 1653int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1654{
1655 switch (msr) {
1656#ifdef CONFIG_X86_64
1657 case MSR_EFER:
1658 set_efer(vcpu, data);
1659 break;
1660#endif
1661 case MSR_IA32_MC0_STATUS:
f0242478 1662 pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
3bab1f5d
AK		 1663 __FUNCTION__, data);
1664 break;
0e5bf0d0 1665 case MSR_IA32_MCG_STATUS:
f0242478 1666 pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
0e5bf0d0
SK		 1667 __FUNCTION__, data);
1668 break;
3bab1f5d
AK		 1669 case MSR_IA32_UCODE_REV:
1670 case MSR_IA32_UCODE_WRITE:
1671 case 0x200 ... 0x2ff: /* MTRRs */
1672 break;
1673 case MSR_IA32_APICBASE:
7017fc3d 1674 kvm_set_apic_base(vcpu, data);
3bab1f5d 1675 break;
6f00e68f
AK		 1676 case MSR_IA32_MISC_ENABLE:
1677 vcpu->ia32_misc_enable_msr = data;
1678 break;
102d8325
IM		 1679 /*
1680 * This is the 'probe whether the host is KVM' logic:
1681 */
1682 case MSR_KVM_API_MAGIC:
1683 return vcpu_register_para(vcpu, data);
1684
3bab1f5d 1685 default:
f0242478 1686 pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr);
3bab1f5d
AK		 1687 return 1;
1688 }
1689 return 0;
1690}
1691EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1692
6aa8b732
AK		 1693/*
1694 * Writes msr value into into the appropriate "register".
1695 * Returns 0 on success, non-0 otherwise.
1696 * Assumes vcpu_load() was already called.
1697 */
35f3f286 1698int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
6aa8b732 1699{
cbdd1bea 1700 return kvm_x86_ops->set_msr(vcpu, msr_index, data);
6aa8b732
AK		 1701}
1702
1703void kvm_resched(struct kvm_vcpu *vcpu)
1704{
3fca0365
YD		 1705 if (!need_resched())
1706 return;
6aa8b732 1707 cond_resched();
6aa8b732
AK		 1708}
1709EXPORT_SYMBOL_GPL(kvm_resched);
1710
06465c5a
AK		 1711void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1712{
1713 int i;
1714 u32 function;
1715 struct kvm_cpuid_entry *e, *best;
1716
cbdd1bea 1717 kvm_x86_ops->cache_regs(vcpu);
06465c5a
AK		 1718 function = vcpu->regs[VCPU_REGS_RAX];
1719 vcpu->regs[VCPU_REGS_RAX] = 0;
1720 vcpu->regs[VCPU_REGS_RBX] = 0;
1721 vcpu->regs[VCPU_REGS_RCX] = 0;
1722 vcpu->regs[VCPU_REGS_RDX] = 0;
1723 best = NULL;
1724 for (i = 0; i < vcpu->cpuid_nent; ++i) {
1725 e = &vcpu->cpuid_entries[i];
1726 if (e->function == function) {
1727 best = e;
1728 break;
1729 }
1730 /*
1731 * Both basic or both extended?
1732 */
1733 if (((e->function ^ function) & 0x80000000) == 0)
1734 if (!best || e->function > best->function)
1735 best = e;
1736 }
1737 if (best) {
1738 vcpu->regs[VCPU_REGS_RAX] = best->eax;
1739 vcpu->regs[VCPU_REGS_RBX] = best->ebx;
1740 vcpu->regs[VCPU_REGS_RCX] = best->ecx;
1741 vcpu->regs[VCPU_REGS_RDX] = best->edx;
1742 }
cbdd1bea
CE		 1743 kvm_x86_ops->decache_regs(vcpu);
1744 kvm_x86_ops->skip_emulated_instruction(vcpu);
06465c5a
AK		 1745}
1746EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
1747
039576c0 1748static int pio_copy_data(struct kvm_vcpu *vcpu)
46fc1477 1749{
039576c0
AK		 1750 void *p = vcpu->pio_data;
1751 void *q;
1752 unsigned bytes;
1753 int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1;
1754
039576c0
AK		 1755 q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE,
1756 PAGE_KERNEL);
1757 if (!q) {
039576c0
AK		 1758 free_pio_guest_pages(vcpu);
1759 return -ENOMEM;
1760 }
1761 q += vcpu->pio.guest_page_offset;
1762 bytes = vcpu->pio.size * vcpu->pio.cur_count;
1763 if (vcpu->pio.in)
1764 memcpy(q, p, bytes);
1765 else
1766 memcpy(p, q, bytes);
1767 q -= vcpu->pio.guest_page_offset;
1768 vunmap(q);
039576c0
AK		 1769 free_pio_guest_pages(vcpu);
1770 return 0;
1771}
1772
1773static int complete_pio(struct kvm_vcpu *vcpu)
1774{
1775 struct kvm_pio_request *io = &vcpu->pio;
46fc1477 1776 long delta;
039576c0 1777 int r;
46fc1477 1778
cbdd1bea 1779 kvm_x86_ops->cache_regs(vcpu);
46fc1477
AK		 1780
1781 if (!io->string) {
039576c0
AK		 1782 if (io->in)
1783 memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data,
46fc1477
AK		 1784 io->size);
1785 } else {
039576c0
AK		 1786 if (io->in) {
1787 r = pio_copy_data(vcpu);
1788 if (r) {
cbdd1bea 1789 kvm_x86_ops->cache_regs(vcpu);
039576c0
AK		 1790 return r;
1791 }
1792 }
1793
46fc1477
AK		 1794 delta = 1;
1795 if (io->rep) {
039576c0 1796 delta *= io->cur_count;
46fc1477
AK		 1797 /*
1798 * The size of the register should really depend on
1799 * current address size.
1800 */
1801 vcpu->regs[VCPU_REGS_RCX] -= delta;
1802 }
039576c0 1803 if (io->down)
46fc1477
AK		 1804 delta = -delta;
1805 delta *= io->size;
039576c0 1806 if (io->in)
46fc1477
AK		 1807 vcpu->regs[VCPU_REGS_RDI] += delta;
1808 else
1809 vcpu->regs[VCPU_REGS_RSI] += delta;
1810 }
1811
cbdd1bea 1812 kvm_x86_ops->decache_regs(vcpu);
46fc1477 1813
039576c0
AK		 1814 io->count -= io->cur_count;
1815 io->cur_count = 0;
1816
1817 if (!io->count)
cbdd1bea 1818 kvm_x86_ops->skip_emulated_instruction(vcpu);
039576c0 1819 return 0;
46fc1477
AK		 1820}
1821
65619eb5
ED		 1822static void kernel_pio(struct kvm_io_device *pio_dev,
1823 struct kvm_vcpu *vcpu,
1824 void *pd)
74906345
ED		 1825{
1826 /* TODO: String I/O for in kernel device */
1827
9cf98828 1828 mutex_lock(&vcpu->kvm->lock);
74906345
ED		 1829 if (vcpu->pio.in)
1830 kvm_iodevice_read(pio_dev, vcpu->pio.port,
1831 vcpu->pio.size,
65619eb5 1832 pd);
74906345
ED		 1833 else
1834 kvm_iodevice_write(pio_dev, vcpu->pio.port,
1835 vcpu->pio.size,
65619eb5 1836 pd);
9cf98828 1837 mutex_unlock(&vcpu->kvm->lock);
65619eb5
ED		 1838}
1839
1840static void pio_string_write(struct kvm_io_device *pio_dev,
1841 struct kvm_vcpu *vcpu)
1842{
1843 struct kvm_pio_request *io = &vcpu->pio;
1844 void *pd = vcpu->pio_data;
1845 int i;
1846
9cf98828 1847 mutex_lock(&vcpu->kvm->lock);
65619eb5
ED		 1848 for (i = 0; i < io->cur_count; i++) {
1849 kvm_iodevice_write(pio_dev, io->port,
1850 io->size,
1851 pd);
1852 pd += io->size;
1853 }
9cf98828 1854 mutex_unlock(&vcpu->kvm->lock);
74906345
ED		 1855}
1856
3090dd73
LV		 1857int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
1858 int size, unsigned port)
1859{
1860 struct kvm_io_device *pio_dev;
1861
1862 vcpu->run->exit_reason = KVM_EXIT_IO;
1863 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
1864 vcpu->run->io.size = vcpu->pio.size = size;
1865 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
1866 vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1;
1867 vcpu->run->io.port = vcpu->pio.port = port;
1868 vcpu->pio.in = in;
1869 vcpu->pio.string = 0;
1870 vcpu->pio.down = 0;
1871 vcpu->pio.guest_page_offset = 0;
1872 vcpu->pio.rep = 0;
1873
cbdd1bea 1874 kvm_x86_ops->cache_regs(vcpu);
3090dd73 1875 memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4);
cbdd1bea 1876 kvm_x86_ops->decache_regs(vcpu);
3090dd73
LV		 1877
1878 pio_dev = vcpu_find_pio_dev(vcpu, port);
1879 if (pio_dev) {
1880 kernel_pio(pio_dev, vcpu, vcpu->pio_data);
1881 complete_pio(vcpu);
1882 return 1;
1883 }
1884 return 0;
1885}
1886EXPORT_SYMBOL_GPL(kvm_emulate_pio);
1887
1888int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in,
1889 int size, unsigned long count, int down,
039576c0
AK		 1890 gva_t address, int rep, unsigned port)
1891{
1892 unsigned now, in_page;
65619eb5 1893 int i, ret = 0;
039576c0
AK		 1894 int nr_pages = 1;
1895 struct page *page;
74906345 1896 struct kvm_io_device *pio_dev;
039576c0
AK		 1897
1898 vcpu->run->exit_reason = KVM_EXIT_IO;
1899 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
3090dd73 1900 vcpu->run->io.size = vcpu->pio.size = size;
039576c0 1901 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
3090dd73
LV		 1902 vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count;
1903 vcpu->run->io.port = vcpu->pio.port = port;
039576c0 1904 vcpu->pio.in = in;
3090dd73 1905 vcpu->pio.string = 1;
039576c0
AK		 1906 vcpu->pio.down = down;
1907 vcpu->pio.guest_page_offset = offset_in_page(address);
1908 vcpu->pio.rep = rep;
1909
039576c0 1910 if (!count) {
cbdd1bea 1911 kvm_x86_ops->skip_emulated_instruction(vcpu);
039576c0
AK		 1912 return 1;
1913 }
1914
039576c0
AK		 1915 if (!down)
1916 in_page = PAGE_SIZE - offset_in_page(address);
1917 else
1918 in_page = offset_in_page(address) + size;
1919 now = min(count, (unsigned long)in_page / size);
1920 if (!now) {
1921 /*
1922 * String I/O straddles page boundary. Pin two guest pages
1923 * so that we satisfy atomicity constraints. Do just one
1924 * transaction to avoid complexity.
1925 */
1926 nr_pages = 2;
1927 now = 1;
1928 }
1929 if (down) {
1930 /*
1931 * String I/O in reverse. Yuck. Kill the guest, fix later.
1932 */
f0242478 1933 pr_unimpl(vcpu, "guest string pio down\n");
039576c0
AK		 1934 inject_gp(vcpu);
1935 return 1;
1936 }
1937 vcpu->run->io.count = now;
1938 vcpu->pio.cur_count = now;
1939
1940 for (i = 0; i < nr_pages; ++i) {
11ec2804 1941 mutex_lock(&vcpu->kvm->lock);
039576c0
AK		 1942 page = gva_to_page(vcpu, address + i * PAGE_SIZE);
1943 if (page)
1944 get_page(page);
1945 vcpu->pio.guest_pages[i] = page;
11ec2804 1946 mutex_unlock(&vcpu->kvm->lock);
039576c0
AK		 1947 if (!page) {
1948 inject_gp(vcpu);
1949 free_pio_guest_pages(vcpu);
1950 return 1;
1951 }
1952 }
1953
3090dd73 1954 pio_dev = vcpu_find_pio_dev(vcpu, port);
65619eb5
ED		 1955 if (!vcpu->pio.in) {
1956 /* string PIO write */
1957 ret = pio_copy_data(vcpu);
1958 if (ret >= 0 && pio_dev) {
1959 pio_string_write(pio_dev, vcpu);
1960 complete_pio(vcpu);
1961 if (vcpu->pio.count == 0)
1962 ret = 1;
1963 }
1964 } else if (pio_dev)
f0242478 1965 pr_unimpl(vcpu, "no string pio read support yet, "
65619eb5
ED		 1966 "port %x size %d count %ld\n",
1967 port, size, count);
1968
1969 return ret;
039576c0 1970}
3090dd73 1971EXPORT_SYMBOL_GPL(kvm_emulate_pio_string);
039576c0 1972
bccf2150 1973static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
6aa8b732 1974{
6aa8b732 1975 int r;
1961d276 1976 sigset_t sigsaved;
6aa8b732 1977
bccf2150 1978 vcpu_load(vcpu);
6aa8b732 1979
c5ec1534
HQ		 1980 if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) {
1981 kvm_vcpu_block(vcpu);
1982 vcpu_put(vcpu);
1983 return -EAGAIN;
1984 }
1985
1961d276
AK		 1986 if (vcpu->sigset_active)
1987 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1988
54810342 1989 /* re-sync apic's tpr */
5cd4f6fd
HQ		 1990 if (!irqchip_in_kernel(vcpu->kvm))
1991 set_cr8(vcpu, kvm_run->cr8);
54810342 1992
02c83209
AK		 1993 if (vcpu->pio.cur_count) {
1994 r = complete_pio(vcpu);
1995 if (r)
1996 goto out;
1997 }
1998
1999 if (vcpu->mmio_needed) {
2000 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
2001 vcpu->mmio_read_completed = 1;
2002 vcpu->mmio_needed = 0;
2003 r = emulate_instruction(vcpu, kvm_run,
2004 vcpu->mmio_fault_cr2, 0);
2005 if (r == EMULATE_DO_MMIO) {
2006 /*
2007 * Read-modify-write. Back to userspace.
2008 */
02c83209
AK		 2009 r = 0;
2010 goto out;
46fc1477 2011 }
6aa8b732
AK		 2012 }
2013
8eb7d334 2014 if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
cbdd1bea 2015 kvm_x86_ops->cache_regs(vcpu);
b4e63f56 2016 vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
cbdd1bea 2017 kvm_x86_ops->decache_regs(vcpu);
b4e63f56
AK		 2018 }
2019
cbdd1bea 2020 r = kvm_x86_ops->run(vcpu, kvm_run);
6aa8b732 2021
039576c0 2022out:
1961d276
AK		 2023 if (vcpu->sigset_active)
2024 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2025
6aa8b732
AK		 2026 vcpu_put(vcpu);
2027 return r;
2028}
2029
bccf2150
AK		 2030static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu,
2031 struct kvm_regs *regs)
6aa8b732 2032{
bccf2150 2033 vcpu_load(vcpu);
6aa8b732 2034
cbdd1bea 2035 kvm_x86_ops->cache_regs(vcpu);
6aa8b732
AK		 2036
2037 regs->rax = vcpu->regs[VCPU_REGS_RAX];
2038 regs->rbx = vcpu->regs[VCPU_REGS_RBX];
2039 regs->rcx = vcpu->regs[VCPU_REGS_RCX];
2040 regs->rdx = vcpu->regs[VCPU_REGS_RDX];
2041 regs->rsi = vcpu->regs[VCPU_REGS_RSI];
2042 regs->rdi = vcpu->regs[VCPU_REGS_RDI];
2043 regs->rsp = vcpu->regs[VCPU_REGS_RSP];
2044 regs->rbp = vcpu->regs[VCPU_REGS_RBP];
05b3e0c2 2045#ifdef CONFIG_X86_64
6aa8b732
AK		 2046 regs->r8 = vcpu->regs[VCPU_REGS_R8];
2047 regs->r9 = vcpu->regs[VCPU_REGS_R9];
2048 regs->r10 = vcpu->regs[VCPU_REGS_R10];
2049 regs->r11 = vcpu->regs[VCPU_REGS_R11];
2050 regs->r12 = vcpu->regs[VCPU_REGS_R12];
2051 regs->r13 = vcpu->regs[VCPU_REGS_R13];
2052 regs->r14 = vcpu->regs[VCPU_REGS_R14];
2053 regs->r15 = vcpu->regs[VCPU_REGS_R15];
2054#endif
2055
2056 regs->rip = vcpu->rip;
cbdd1bea 2057 regs->rflags = kvm_x86_ops->get_rflags(vcpu);
6aa8b732
AK		 2058
2059 /*
2060 * Don't leak debug flags in case they were set for guest debugging
2061 */
2062 if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
2063 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
2064
2065 vcpu_put(vcpu);
2066
2067 return 0;
2068}
2069
bccf2150
AK		 2070static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu,
2071 struct kvm_regs *regs)
6aa8b732 2072{
bccf2150 2073 vcpu_load(vcpu);
6aa8b732
AK		 2074
2075 vcpu->regs[VCPU_REGS_RAX] = regs->rax;
2076 vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
2077 vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
2078 vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
2079 vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
2080 vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
2081 vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
2082 vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
05b3e0c2 2083#ifdef CONFIG_X86_64
6aa8b732
AK		 2084 vcpu->regs[VCPU_REGS_R8] = regs->r8;
2085 vcpu->regs[VCPU_REGS_R9] = regs->r9;
2086 vcpu->regs[VCPU_REGS_R10] = regs->r10;
2087 vcpu->regs[VCPU_REGS_R11] = regs->r11;
2088 vcpu->regs[VCPU_REGS_R12] = regs->r12;
2089 vcpu->regs[VCPU_REGS_R13] = regs->r13;
2090 vcpu->regs[VCPU_REGS_R14] = regs->r14;
2091 vcpu->regs[VCPU_REGS_R15] = regs->r15;
2092#endif
2093
2094 vcpu->rip = regs->rip;
cbdd1bea 2095 kvm_x86_ops->set_rflags(vcpu, regs->rflags);
6aa8b732 2096
cbdd1bea 2097 kvm_x86_ops->decache_regs(vcpu);
6aa8b732
AK		 2098
2099 vcpu_put(vcpu);
2100
2101 return 0;
2102}
2103
2104static void get_segment(struct kvm_vcpu *vcpu,
2105 struct kvm_segment *var, int seg)
2106{
cbdd1bea 2107 return kvm_x86_ops->get_segment(vcpu, var, seg);
6aa8b732
AK		 2108}
2109
bccf2150
AK		 2110static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2111 struct kvm_sregs *sregs)
6aa8b732 2112{
6aa8b732 2113 struct descriptor_table dt;
2a8067f1 2114 int pending_vec;
6aa8b732 2115
bccf2150 2116 vcpu_load(vcpu);
6aa8b732
AK		 2117
2118 get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2119 get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2120 get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2121 get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2122 get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2123 get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2124
2125 get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2126 get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2127
cbdd1bea 2128 kvm_x86_ops->get_idt(vcpu, &dt);
6aa8b732
AK		 2129 sregs->idt.limit = dt.limit;
2130 sregs->idt.base = dt.base;
cbdd1bea 2131 kvm_x86_ops->get_gdt(vcpu, &dt);
6aa8b732
AK		 2132 sregs->gdt.limit = dt.limit;
2133 sregs->gdt.base = dt.base;
2134
cbdd1bea 2135 kvm_x86_ops->decache_cr4_guest_bits(vcpu);
6aa8b732
AK		 2136 sregs->cr0 = vcpu->cr0;
2137 sregs->cr2 = vcpu->cr2;
2138 sregs->cr3 = vcpu->cr3;
2139 sregs->cr4 = vcpu->cr4;
7017fc3d 2140 sregs->cr8 = get_cr8(vcpu);
6aa8b732 2141 sregs->efer = vcpu->shadow_efer;
7017fc3d 2142 sregs->apic_base = kvm_get_apic_base(vcpu);
6aa8b732 2143
2a8067f1 2144 if (irqchip_in_kernel(vcpu->kvm)) {
c52fb35a
HQ		 2145 memset(sregs->interrupt_bitmap, 0,
2146 sizeof sregs->interrupt_bitmap);
cbdd1bea 2147 pending_vec = kvm_x86_ops->get_irq(vcpu);
2a8067f1
ED		 2148 if (pending_vec >= 0)
2149 set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap);
2150 } else
c52fb35a
HQ		 2151 memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
2152 sizeof sregs->interrupt_bitmap);
6aa8b732
AK		 2153
2154 vcpu_put(vcpu);
2155
2156 return 0;
2157}
2158
2159static void set_segment(struct kvm_vcpu *vcpu,
2160 struct kvm_segment *var, int seg)
2161{
cbdd1bea 2162 return kvm_x86_ops->set_segment(vcpu, var, seg);
6aa8b732
AK		 2163}
2164
bccf2150
AK		 2165static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2166 struct kvm_sregs *sregs)
6aa8b732 2167{
6aa8b732 2168 int mmu_reset_needed = 0;
2a8067f1 2169 int i, pending_vec, max_bits;
6aa8b732
AK		 2170 struct descriptor_table dt;
2171
bccf2150 2172 vcpu_load(vcpu);
6aa8b732 2173
6aa8b732
AK		 2174 dt.limit = sregs->idt.limit;
2175 dt.base = sregs->idt.base;
cbdd1bea 2176 kvm_x86_ops->set_idt(vcpu, &dt);
6aa8b732
AK		 2177 dt.limit = sregs->gdt.limit;
2178 dt.base = sregs->gdt.base;
cbdd1bea 2179 kvm_x86_ops->set_gdt(vcpu, &dt);
6aa8b732
AK		 2180
2181 vcpu->cr2 = sregs->cr2;
2182 mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
2183 vcpu->cr3 = sregs->cr3;
2184
7017fc3d 2185 set_cr8(vcpu, sregs->cr8);
6aa8b732
AK		 2186
2187 mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
05b3e0c2 2188#ifdef CONFIG_X86_64
cbdd1bea 2189 kvm_x86_ops->set_efer(vcpu, sregs->efer);
6aa8b732 2190#endif
7017fc3d 2191 kvm_set_apic_base(vcpu, sregs->apic_base);
6aa8b732 2192
cbdd1bea 2193 kvm_x86_ops->decache_cr4_guest_bits(vcpu);
399badf3 2194
6aa8b732 2195 mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
81f50e3b 2196 vcpu->cr0 = sregs->cr0;
cbdd1bea 2197 kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
6aa8b732
AK		 2198
2199 mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
cbdd1bea 2200 kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
1b0973bd
AK		 2201 if (!is_long_mode(vcpu) && is_pae(vcpu))
2202 load_pdptrs(vcpu, vcpu->cr3);
6aa8b732
AK		 2203
2204 if (mmu_reset_needed)
2205 kvm_mmu_reset_context(vcpu);
2206
c52fb35a
HQ		 2207 if (!irqchip_in_kernel(vcpu->kvm)) {
2208 memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
2209 sizeof vcpu->irq_pending);
2210 vcpu->irq_summary = 0;
2211 for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i)
2212 if (vcpu->irq_pending[i])
2213 __set_bit(i, &vcpu->irq_summary);
2a8067f1
ED		 2214 } else {
2215 max_bits = (sizeof sregs->interrupt_bitmap) << 3;
2216 pending_vec = find_first_bit(
2217 (const unsigned long *)sregs->interrupt_bitmap,
2218 max_bits);
2219 /* Only pending external irq is handled here */
2220 if (pending_vec < max_bits) {
cbdd1bea 2221 kvm_x86_ops->set_irq(vcpu, pending_vec);
2a8067f1
ED		 2222 printk("Set back pending irq %d\n", pending_vec);
2223 }
c52fb35a 2224 }
6aa8b732 2225
024aa1c0
AK		 2226 set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
2227 set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
2228 set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
2229 set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
2230 set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
2231 set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
2232
2233 set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
2234 set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
2235
6aa8b732
AK		 2236 vcpu_put(vcpu);
2237
2238 return 0;
2239}
2240
1747fb71
RR		 2241void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
2242{
2243 struct kvm_segment cs;
2244
2245 get_segment(vcpu, &cs, VCPU_SREG_CS);
2246 *db = cs.db;
2247 *l = cs.l;
2248}
2249EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
2250
6aa8b732
AK		 2251/*
2252 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
2253 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
bf591b24
MR		 2254 *
2255 * This list is modified at module load time to reflect the
2256 * capabilities of the host cpu.
6aa8b732
AK		 2257 */
2258static u32 msrs_to_save[] = {
2259 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
2260 MSR_K6_STAR,
05b3e0c2 2261#ifdef CONFIG_X86_64
6aa8b732
AK		 2262 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
2263#endif
2264 MSR_IA32_TIME_STAMP_COUNTER,
2265};
2266
bf591b24
MR		 2267static unsigned num_msrs_to_save;
2268
6f00e68f
AK		 2269static u32 emulated_msrs[] = {
2270 MSR_IA32_MISC_ENABLE,
2271};
2272
bf591b24
MR		 2273static __init void kvm_init_msr_list(void)
2274{
2275 u32 dummy[2];
2276 unsigned i, j;
2277
2278 for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
2279 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
2280 continue;
2281 if (j < i)
2282 msrs_to_save[j] = msrs_to_save[i];
2283 j++;
2284 }
2285 num_msrs_to_save = j;
2286}
6aa8b732
AK		 2287
2288/*
2289 * Adapt set_msr() to msr_io()'s calling convention
2290 */
2291static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2292{
35f3f286 2293 return kvm_set_msr(vcpu, index, *data);
6aa8b732
AK		 2294}
2295
2296/*
2297 * Read or write a bunch of msrs. All parameters are kernel addresses.
2298 *
2299 * @return number of msrs set successfully.
2300 */
bccf2150 2301static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
6aa8b732
AK		 2302 struct kvm_msr_entry *entries,
2303 int (*do_msr)(struct kvm_vcpu *vcpu,
2304 unsigned index, u64 *data))
2305{
6aa8b732
AK		 2306 int i;
2307
bccf2150 2308 vcpu_load(vcpu);
6aa8b732
AK		 2309
2310 for (i = 0; i < msrs->nmsrs; ++i)
2311 if (do_msr(vcpu, entries[i].index, &entries[i].data))
2312 break;
2313
2314 vcpu_put(vcpu);
2315
2316 return i;
2317}
2318
2319/*
2320 * Read or write a bunch of msrs. Parameters are user addresses.
2321 *
2322 * @return number of msrs set successfully.
2323 */
bccf2150 2324static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
6aa8b732
AK		 2325 int (*do_msr)(struct kvm_vcpu *vcpu,
2326 unsigned index, u64 *data),
2327 int writeback)
2328{
2329 struct kvm_msrs msrs;
2330 struct kvm_msr_entry *entries;
2331 int r, n;
2332 unsigned size;
2333
2334 r = -EFAULT;
2335 if (copy_from_user(&msrs, user_msrs, sizeof msrs))
2336 goto out;
2337
2338 r = -E2BIG;
2339 if (msrs.nmsrs >= MAX_IO_MSRS)
2340 goto out;
2341
2342 r = -ENOMEM;
2343 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
2344 entries = vmalloc(size);
2345 if (!entries)
2346 goto out;
2347
2348 r = -EFAULT;
2349 if (copy_from_user(entries, user_msrs->entries, size))
2350 goto out_free;
2351
bccf2150 2352 r = n = __msr_io(vcpu, &msrs, entries, do_msr);
6aa8b732
AK		 2353 if (r < 0)
2354 goto out_free;
2355
2356 r = -EFAULT;
2357 if (writeback && copy_to_user(user_msrs->entries, entries, size))
2358 goto out_free;
2359
2360 r = n;
2361
2362out_free:
2363 vfree(entries);
2364out:
2365 return r;
2366}
2367
2368/*
2369 * Translate a guest virtual address to a guest physical address.
2370 */
bccf2150
AK		 2371static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2372 struct kvm_translation *tr)
6aa8b732
AK		 2373{
2374 unsigned long vaddr = tr->linear_address;
6aa8b732
AK		 2375 gpa_t gpa;
2376
bccf2150 2377 vcpu_load(vcpu);
11ec2804 2378 mutex_lock(&vcpu->kvm->lock);
6aa8b732
AK		 2379 gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
2380 tr->physical_address = gpa;
2381 tr->valid = gpa != UNMAPPED_GVA;
2382 tr->writeable = 1;
2383 tr->usermode = 0;
11ec2804 2384 mutex_unlock(&vcpu->kvm->lock);
6aa8b732
AK		 2385 vcpu_put(vcpu);
2386
2387 return 0;
2388}
2389
bccf2150
AK		 2390static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
2391 struct kvm_interrupt *irq)
6aa8b732 2392{
6aa8b732
AK		 2393 if (irq->irq < 0 || irq->irq >= 256)
2394 return -EINVAL;
97222cc8
ED		 2395 if (irqchip_in_kernel(vcpu->kvm))
2396 return -ENXIO;
bccf2150 2397 vcpu_load(vcpu);
6aa8b732
AK		 2398
2399 set_bit(irq->irq, vcpu->irq_pending);
2400 set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
2401
2402 vcpu_put(vcpu);
2403
2404 return 0;
2405}
2406
bccf2150
AK		 2407static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
2408 struct kvm_debug_guest *dbg)
6aa8b732 2409{
6aa8b732
AK		 2410 int r;
2411
bccf2150 2412 vcpu_load(vcpu);
6aa8b732 2413
cbdd1bea 2414 r = kvm_x86_ops->set_guest_debug(vcpu, dbg);
6aa8b732
AK		 2415
2416 vcpu_put(vcpu);
2417
2418 return r;
2419}
2420
9a2bb7f4
AK		 2421static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
2422 unsigned long address,
2423 int *type)
2424{
2425 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2426 unsigned long pgoff;
2427 struct page *page;
2428
9a2bb7f4 2429 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
039576c0
AK		 2430 if (pgoff == 0)
2431 page = virt_to_page(vcpu->run);
2432 else if (pgoff == KVM_PIO_PAGE_OFFSET)
2433 page = virt_to_page(vcpu->pio_data);
2434 else
9a2bb7f4 2435 return NOPAGE_SIGBUS;
9a2bb7f4 2436 get_page(page);
cd0d9137
NAQ		 2437 if (type != NULL)
2438 *type = VM_FAULT_MINOR;
2439
9a2bb7f4
AK		 2440 return page;
2441}
2442
2443static struct vm_operations_struct kvm_vcpu_vm_ops = {
2444 .nopage = kvm_vcpu_nopage,
2445};
2446
2447static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2448{
2449 vma->vm_ops = &kvm_vcpu_vm_ops;
2450 return 0;
2451}
2452
bccf2150
AK		 2453static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2454{
2455 struct kvm_vcpu *vcpu = filp->private_data;
2456
2457 fput(vcpu->kvm->filp);
2458 return 0;
2459}
2460
2461static struct file_operations kvm_vcpu_fops = {
2462 .release = kvm_vcpu_release,
2463 .unlocked_ioctl = kvm_vcpu_ioctl,
2464 .compat_ioctl = kvm_vcpu_ioctl,
9a2bb7f4 2465 .mmap = kvm_vcpu_mmap,
bccf2150
AK		 2466};
2467
2468/*
2469 * Allocates an inode for the vcpu.
2470 */
2471static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2472{
2473 int fd, r;
2474 struct inode *inode;
2475 struct file *file;
2476
d6d28168
AK		 2477 r = anon_inode_getfd(&fd, &inode, &file,
2478 "kvm-vcpu", &kvm_vcpu_fops, vcpu);
2479 if (r)
2480 return r;
bccf2150 2481 atomic_inc(&vcpu->kvm->filp->f_count);
bccf2150 2482 return fd;
bccf2150
AK		 2483}
2484
c5ea7660
AK		 2485/*
2486 * Creates some virtual cpus. Good luck creating more than one.
2487 */
2488static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
2489{
2490 int r;
2491 struct kvm_vcpu *vcpu;
2492
c5ea7660 2493 if (!valid_vcpu(n))
fb3f0f51 2494 return -EINVAL;
c5ea7660 2495
cbdd1bea 2496 vcpu = kvm_x86_ops->vcpu_create(kvm, n);
fb3f0f51
RR		 2497 if (IS_ERR(vcpu))
2498 return PTR_ERR(vcpu);
c5ea7660 2499
15ad7146
AK		 2500 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2501
b114b080
RR		 2502 /* We do fxsave: this must be aligned. */
2503 BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF);
2504
fb3f0f51 2505 vcpu_load(vcpu);
c5ea7660 2506 r = kvm_mmu_setup(vcpu);
c5ea7660 2507 vcpu_put(vcpu);
c5ea7660 2508 if (r < 0)
fb3f0f51
RR		 2509 goto free_vcpu;
2510
11ec2804 2511 mutex_lock(&kvm->lock);
fb3f0f51
RR		 2512 if (kvm->vcpus[n]) {
2513 r = -EEXIST;
11ec2804 2514 mutex_unlock(&kvm->lock);
fb3f0f51
RR		 2515 goto mmu_unload;
2516 }
2517 kvm->vcpus[n] = vcpu;
11ec2804 2518 mutex_unlock(&kvm->lock);
c5ea7660 2519
fb3f0f51 2520 /* Now it's all set up, let userspace reach it */
bccf2150
AK		 2521 r = create_vcpu_fd(vcpu);
2522 if (r < 0)
fb3f0f51
RR		 2523 goto unlink;
2524 return r;
39c3b86e 2525
fb3f0f51 2526unlink:
11ec2804 2527 mutex_lock(&kvm->lock);
fb3f0f51 2528 kvm->vcpus[n] = NULL;
11ec2804 2529 mutex_unlock(&kvm->lock);
a2fa3e9f 2530
fb3f0f51
RR		 2531mmu_unload:
2532 vcpu_load(vcpu);
2533 kvm_mmu_unload(vcpu);
2534 vcpu_put(vcpu);
c5ea7660 2535
fb3f0f51 2536free_vcpu:
cbdd1bea 2537 kvm_x86_ops->vcpu_free(vcpu);
c5ea7660
AK		 2538 return r;
2539}
2540
2cc51560
ED		 2541static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
2542{
2543 u64 efer;
2544 int i;
2545 struct kvm_cpuid_entry *e, *entry;
2546
2547 rdmsrl(MSR_EFER, efer);
2548 entry = NULL;
2549 for (i = 0; i < vcpu->cpuid_nent; ++i) {
2550 e = &vcpu->cpuid_entries[i];
2551 if (e->function == 0x80000001) {
2552 entry = e;
2553 break;
2554 }
2555 }
4c981b43 2556 if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) {
2cc51560 2557 entry->edx &= ~(1 << 20);
4c981b43 2558 printk(KERN_INFO "kvm: guest NX capability removed\n");
2cc51560
ED		 2559 }
2560}
2561
06465c5a
AK		 2562static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
2563 struct kvm_cpuid *cpuid,
2564 struct kvm_cpuid_entry __user *entries)
2565{
2566 int r;
2567
2568 r = -E2BIG;
2569 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
2570 goto out;
2571 r = -EFAULT;
2572 if (copy_from_user(&vcpu->cpuid_entries, entries,
2573 cpuid->nent * sizeof(struct kvm_cpuid_entry)))
2574 goto out;
2575 vcpu->cpuid_nent = cpuid->nent;
2cc51560 2576 cpuid_fix_nx_cap(vcpu);
06465c5a
AK		 2577 return 0;
2578
2579out:
2580 return r;
2581}
2582
1961d276
AK		 2583static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2584{
2585 if (sigset) {
2586 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2587 vcpu->sigset_active = 1;
2588 vcpu->sigset = *sigset;
2589 } else
2590 vcpu->sigset_active = 0;
2591 return 0;
2592}
2593
b8836737
AK		 2594/*
2595 * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
2596 * we have asm/x86/processor.h
2597 */
2598struct fxsave {
2599 u16 cwd;
2600 u16 swd;
2601 u16 twd;
2602 u16 fop;
2603 u64 rip;
2604 u64 rdp;
2605 u32 mxcsr;
2606 u32 mxcsr_mask;
2607 u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
2608#ifdef CONFIG_X86_64
2609 u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
2610#else
2611 u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
2612#endif
2613};
2614
2615static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2616{
b114b080 2617 struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
b8836737
AK		 2618
2619 vcpu_load(vcpu);
2620
2621 memcpy(fpu->fpr, fxsave->st_space, 128);
2622 fpu->fcw = fxsave->cwd;
2623 fpu->fsw = fxsave->swd;
2624 fpu->ftwx = fxsave->twd;
2625 fpu->last_opcode = fxsave->fop;
2626 fpu->last_ip = fxsave->rip;
2627 fpu->last_dp = fxsave->rdp;
2628 memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
2629
2630 vcpu_put(vcpu);
2631
2632 return 0;
2633}
2634
2635static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2636{
b114b080 2637 struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image;
b8836737
AK		 2638
2639 vcpu_load(vcpu);
2640
2641 memcpy(fxsave->st_space, fpu->fpr, 128);
2642 fxsave->cwd = fpu->fcw;
2643 fxsave->swd = fpu->fsw;
2644 fxsave->twd = fpu->ftwx;
2645 fxsave->fop = fpu->last_opcode;
2646 fxsave->rip = fpu->last_ip;
2647 fxsave->rdp = fpu->last_dp;
2648 memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
2649
2650 vcpu_put(vcpu);
2651
2652 return 0;
2653}
2654
96ad2cc6
ED		 2655static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
2656 struct kvm_lapic_state *s)
2657{
2658 vcpu_load(vcpu);
2659 memcpy(s->regs, vcpu->apic->regs, sizeof *s);
2660 vcpu_put(vcpu);
2661
2662 return 0;
2663}
2664
2665static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
2666 struct kvm_lapic_state *s)
2667{
2668 vcpu_load(vcpu);
2669 memcpy(vcpu->apic->regs, s->regs, sizeof *s);
2670 kvm_apic_post_state_restore(vcpu);
2671 vcpu_put(vcpu);
2672
2673 return 0;
2674}
2675
bccf2150
AK		 2676static long kvm_vcpu_ioctl(struct file *filp,
2677 unsigned int ioctl, unsigned long arg)
6aa8b732 2678{
bccf2150 2679 struct kvm_vcpu *vcpu = filp->private_data;
2f366987 2680 void __user *argp = (void __user *)arg;
6aa8b732
AK		 2681 int r = -EINVAL;
2682
2683 switch (ioctl) {
9a2bb7f4 2684 case KVM_RUN:
f0fe5108
AK		 2685 r = -EINVAL;
2686 if (arg)
2687 goto out;
9a2bb7f4 2688 r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
6aa8b732 2689 break;
6aa8b732
AK		 2690 case KVM_GET_REGS: {
2691 struct kvm_regs kvm_regs;
2692
bccf2150
AK		 2693 memset(&kvm_regs, 0, sizeof kvm_regs);
2694 r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs);
6aa8b732
AK		 2695 if (r)
2696 goto out;
2697 r = -EFAULT;
2f366987 2698 if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs))
6aa8b732
AK		 2699 goto out;
2700 r = 0;
2701 break;
2702 }
2703 case KVM_SET_REGS: {
2704 struct kvm_regs kvm_regs;
2705
2706 r = -EFAULT;
2f366987 2707 if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs))
6aa8b732 2708 goto out;
bccf2150 2709 r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs);
6aa8b732
AK		 2710 if (r)
2711 goto out;
2712 r = 0;
2713 break;
2714 }
2715 case KVM_GET_SREGS: {
2716 struct kvm_sregs kvm_sregs;
2717
bccf2150
AK		 2718 memset(&kvm_sregs, 0, sizeof kvm_sregs);
2719 r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
6aa8b732
AK		 2720 if (r)
2721 goto out;
2722 r = -EFAULT;
2f366987 2723 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
6aa8b732
AK		 2724 goto out;
2725 r = 0;
2726 break;
2727 }
2728 case KVM_SET_SREGS: {
2729 struct kvm_sregs kvm_sregs;
2730
2731 r = -EFAULT;
2f366987 2732 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
6aa8b732 2733 goto out;
bccf2150 2734 r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
6aa8b732
AK		 2735 if (r)
2736 goto out;
2737 r = 0;
2738 break;
2739 }
2740 case KVM_TRANSLATE: {
2741 struct kvm_translation tr;
2742
2743 r = -EFAULT;
2f366987 2744 if (copy_from_user(&tr, argp, sizeof tr))
6aa8b732 2745 goto out;
bccf2150 2746 r = kvm_vcpu_ioctl_translate(vcpu, &tr);
6aa8b732
AK		 2747 if (r)
2748 goto out;
2749 r = -EFAULT;
2f366987 2750 if (copy_to_user(argp, &tr, sizeof tr))
6aa8b732
AK		 2751 goto out;
2752 r = 0;
2753 break;
2754 }
2755 case KVM_INTERRUPT: {
2756 struct kvm_interrupt irq;
2757
2758 r = -EFAULT;
2f366987 2759 if (copy_from_user(&irq, argp, sizeof irq))
6aa8b732 2760 goto out;
bccf2150 2761 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
6aa8b732
AK		 2762 if (r)
2763 goto out;
2764 r = 0;
2765 break;
2766 }
2767 case KVM_DEBUG_GUEST: {
2768 struct kvm_debug_guest dbg;
2769
2770 r = -EFAULT;
2f366987 2771 if (copy_from_user(&dbg, argp, sizeof dbg))
6aa8b732 2772 goto out;
bccf2150 2773 r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg);
6aa8b732
AK		 2774 if (r)
2775 goto out;
2776 r = 0;
2777 break;
2778 }
bccf2150 2779 case KVM_GET_MSRS:
35f3f286 2780 r = msr_io(vcpu, argp, kvm_get_msr, 1);
bccf2150
AK		 2781 break;
2782 case KVM_SET_MSRS:
2783 r = msr_io(vcpu, argp, do_set_msr, 0);
2784 break;
06465c5a
AK		 2785 case KVM_SET_CPUID: {
2786 struct kvm_cpuid __user *cpuid_arg = argp;
2787 struct kvm_cpuid cpuid;
2788
2789 r = -EFAULT;
2790 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
2791 goto out;
2792 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
2793 if (r)
2794 goto out;
2795 break;
2796 }
1961d276
AK		 2797 case KVM_SET_SIGNAL_MASK: {
2798 struct kvm_signal_mask __user *sigmask_arg = argp;
2799 struct kvm_signal_mask kvm_sigmask;
2800 sigset_t sigset, *p;
2801
2802 p = NULL;
2803 if (argp) {
2804 r = -EFAULT;
2805 if (copy_from_user(&kvm_sigmask, argp,
2806 sizeof kvm_sigmask))
2807 goto out;
2808 r = -EINVAL;
2809 if (kvm_sigmask.len != sizeof sigset)
2810 goto out;
2811 r = -EFAULT;
2812 if (copy_from_user(&sigset, sigmask_arg->sigset,
2813 sizeof sigset))
2814 goto out;
2815 p = &sigset;
2816 }
2817 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2818 break;
2819 }
b8836737
AK		 2820 case KVM_GET_FPU: {
2821 struct kvm_fpu fpu;
2822
2823 memset(&fpu, 0, sizeof fpu);
2824 r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu);
2825 if (r)
2826 goto out;
2827 r = -EFAULT;
2828 if (copy_to_user(argp, &fpu, sizeof fpu))
2829 goto out;
2830 r = 0;
2831 break;
2832 }
2833 case KVM_SET_FPU: {
2834 struct kvm_fpu fpu;
2835
2836 r = -EFAULT;
2837 if (copy_from_user(&fpu, argp, sizeof fpu))
2838 goto out;
2839 r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu);
2840 if (r)
2841 goto out;
2842 r = 0;
2843 break;
2844 }
96ad2cc6
ED		 2845 case KVM_GET_LAPIC: {
2846 struct kvm_lapic_state lapic;
2847
2848 memset(&lapic, 0, sizeof lapic);
2849 r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic);
2850 if (r)
2851 goto out;
2852 r = -EFAULT;
2853 if (copy_to_user(argp, &lapic, sizeof lapic))
2854 goto out;
2855 r = 0;
2856 break;
2857 }
2858 case KVM_SET_LAPIC: {
2859 struct kvm_lapic_state lapic;
2860
2861 r = -EFAULT;
2862 if (copy_from_user(&lapic, argp, sizeof lapic))
2863 goto out;
2864 r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);;
2865 if (r)
2866 goto out;
2867 r = 0;
2868 break;
2869 }
bccf2150
AK		 2870 default:
2871 ;
2872 }
2873out:
2874 return r;
2875}
2876
2877static long kvm_vm_ioctl(struct file *filp,
2878 unsigned int ioctl, unsigned long arg)
2879{
2880 struct kvm *kvm = filp->private_data;
2881 void __user *argp = (void __user *)arg;
2882 int r = -EINVAL;
2883
2884 switch (ioctl) {
2885 case KVM_CREATE_VCPU:
2886 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2887 if (r < 0)
2888 goto out;
2889 break;
6aa8b732
AK		 2890 case KVM_SET_MEMORY_REGION: {
2891 struct kvm_memory_region kvm_mem;
2892
2893 r = -EFAULT;
2f366987 2894 if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
6aa8b732 2895 goto out;
2c6f5df9 2896 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem);
6aa8b732
AK		 2897 if (r)
2898 goto out;
2899 break;
2900 }
2901 case KVM_GET_DIRTY_LOG: {
2902 struct kvm_dirty_log log;
2903
2904 r = -EFAULT;
2f366987 2905 if (copy_from_user(&log, argp, sizeof log))
6aa8b732 2906 goto out;
2c6f5df9 2907 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
6aa8b732
AK		 2908 if (r)
2909 goto out;
2910 break;
2911 }
e8207547
AK		 2912 case KVM_SET_MEMORY_ALIAS: {
2913 struct kvm_memory_alias alias;
2914
2915 r = -EFAULT;
2916 if (copy_from_user(&alias, argp, sizeof alias))
2917 goto out;
2918 r = kvm_vm_ioctl_set_memory_alias(kvm, &alias);
2919 if (r)
2920 goto out;
2921 break;
2922 }
85f455f7
ED		 2923 case KVM_CREATE_IRQCHIP:
2924 r = -ENOMEM;
2925 kvm->vpic = kvm_create_pic(kvm);
1fd4f2a5
ED		 2926 if (kvm->vpic) {
2927 r = kvm_ioapic_init(kvm);
2928 if (r) {
2929 kfree(kvm->vpic);
2930 kvm->vpic = NULL;
2931 goto out;
2932 }
2933 }
85f455f7
ED		 2934 else
2935 goto out;
2936 break;
2937 case KVM_IRQ_LINE: {
2938 struct kvm_irq_level irq_event;
2939
2940 r = -EFAULT;
2941 if (copy_from_user(&irq_event, argp, sizeof irq_event))
2942 goto out;
2943 if (irqchip_in_kernel(kvm)) {
9cf98828 2944 mutex_lock(&kvm->lock);
85f455f7
ED		 2945 if (irq_event.irq < 16)
2946 kvm_pic_set_irq(pic_irqchip(kvm),
2947 irq_event.irq,
2948 irq_event.level);
1fd4f2a5
ED		 2949 kvm_ioapic_set_irq(kvm->vioapic,
2950 irq_event.irq,
2951 irq_event.level);
9cf98828 2952 mutex_unlock(&kvm->lock);
85f455f7
ED		 2953 r = 0;
2954 }
2955 break;
2956 }
6ceb9d79
HQ		 2957 case KVM_GET_IRQCHIP: {
2958 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
2959 struct kvm_irqchip chip;
2960
2961 r = -EFAULT;
2962 if (copy_from_user(&chip, argp, sizeof chip))
2963 goto out;
2964 r = -ENXIO;
2965 if (!irqchip_in_kernel(kvm))
2966 goto out;
2967 r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
2968 if (r)
2969 goto out;
2970 r = -EFAULT;
2971 if (copy_to_user(argp, &chip, sizeof chip))
2972 goto out;
2973 r = 0;
2974 break;
2975 }
2976 case KVM_SET_IRQCHIP: {
2977 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
2978 struct kvm_irqchip chip;
2979
2980 r = -EFAULT;
2981 if (copy_from_user(&chip, argp, sizeof chip))
2982 goto out;
2983 r = -ENXIO;
2984 if (!irqchip_in_kernel(kvm))
2985 goto out;
2986 r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
2987 if (r)
2988 goto out;
2989 r = 0;
2990 break;
2991 }
f17abe9a
AK		 2992 default:
2993 ;
2994 }
2995out:
2996 return r;
2997}
2998
2999static struct page *kvm_vm_nopage(struct vm_area_struct *vma,
3000 unsigned long address,
3001 int *type)
3002{
3003 struct kvm *kvm = vma->vm_file->private_data;
3004 unsigned long pgoff;
f17abe9a
AK		 3005 struct page *page;
3006
f17abe9a 3007 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
954bbbc2 3008 page = gfn_to_page(kvm, pgoff);
f17abe9a
AK		 3009 if (!page)
3010 return NOPAGE_SIGBUS;
3011 get_page(page);
cd0d9137
NAQ		 3012 if (type != NULL)
3013 *type = VM_FAULT_MINOR;
3014
f17abe9a
AK		 3015 return page;
3016}
3017
3018static struct vm_operations_struct kvm_vm_vm_ops = {
3019 .nopage = kvm_vm_nopage,
3020};
3021
3022static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
3023{
3024 vma->vm_ops = &kvm_vm_vm_ops;
3025 return 0;
3026}
3027
3028static struct file_operations kvm_vm_fops = {
3029 .release = kvm_vm_release,
3030 .unlocked_ioctl = kvm_vm_ioctl,
3031 .compat_ioctl = kvm_vm_ioctl,
3032 .mmap = kvm_vm_mmap,
3033};
3034
3035static int kvm_dev_ioctl_create_vm(void)
3036{
3037 int fd, r;
3038 struct inode *inode;
3039 struct file *file;
3040 struct kvm *kvm;
3041
f17abe9a 3042 kvm = kvm_create_vm();
d6d28168
AK		 3043 if (IS_ERR(kvm))
3044 return PTR_ERR(kvm);
3045 r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm);
3046 if (r) {
3047 kvm_destroy_vm(kvm);
3048 return r;
f17abe9a
AK		 3049 }
3050
bccf2150 3051 kvm->filp = file;
f17abe9a 3052
f17abe9a 3053 return fd;
f17abe9a
AK		 3054}
3055
3056static long kvm_dev_ioctl(struct file *filp,
3057 unsigned int ioctl, unsigned long arg)
3058{
3059 void __user *argp = (void __user *)arg;
07c45a36 3060 long r = -EINVAL;
f17abe9a
AK		 3061
3062 switch (ioctl) {
3063 case KVM_GET_API_VERSION:
f0fe5108
AK		 3064 r = -EINVAL;
3065 if (arg)
3066 goto out;
f17abe9a
AK		 3067 r = KVM_API_VERSION;
3068 break;
3069 case KVM_CREATE_VM:
f0fe5108
AK		 3070 r = -EINVAL;
3071 if (arg)
3072 goto out;
f17abe9a
AK		 3073 r = kvm_dev_ioctl_create_vm();
3074 break;
6aa8b732 3075 case KVM_GET_MSR_INDEX_LIST: {
2f366987 3076 struct kvm_msr_list __user *user_msr_list = argp;
6aa8b732
AK		 3077 struct kvm_msr_list msr_list;
3078 unsigned n;
3079
3080 r = -EFAULT;
3081 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
3082 goto out;
3083 n = msr_list.nmsrs;
6f00e68f 3084 msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
6aa8b732
AK		 3085 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
3086 goto out;
3087 r = -E2BIG;
bf591b24 3088 if (n < num_msrs_to_save)
6aa8b732
AK		 3089 goto out;
3090 r = -EFAULT;
3091 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
bf591b24 3092 num_msrs_to_save * sizeof(u32)))
6aa8b732 3093 goto out;
6f00e68f
AK		 3094 if (copy_to_user(user_msr_list->indices
3095 + num_msrs_to_save * sizeof(u32),
3096 &emulated_msrs,
3097 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
3098 goto out;
6aa8b732 3099 r = 0;
cc1d8955 3100 break;
6aa8b732 3101 }
85f455f7
ED		 3102 case KVM_CHECK_EXTENSION: {
3103 int ext = (long)argp;
3104
3105 switch (ext) {
3106 case KVM_CAP_IRQCHIP:
b6958ce4 3107 case KVM_CAP_HLT:
85f455f7
ED		 3108 r = 1;
3109 break;
3110 default:
3111 r = 0;
3112 break;
3113 }
5d308f45 3114 break;
85f455f7 3115 }
07c45a36
AK		 3116 case KVM_GET_VCPU_MMAP_SIZE:
3117 r = -EINVAL;
3118 if (arg)
3119 goto out;
039576c0 3120 r = 2 * PAGE_SIZE;
07c45a36 3121 break;
6aa8b732
AK		 3122 default:
3123 ;
3124 }
3125out:
3126 return r;
3127}
3128
6aa8b732 3129static struct file_operations kvm_chardev_ops = {
6aa8b732
AK		 3130 .unlocked_ioctl = kvm_dev_ioctl,
3131 .compat_ioctl = kvm_dev_ioctl,
6aa8b732
AK		 3132};
3133
3134static struct miscdevice kvm_dev = {
bbe4432e 3135 KVM_MINOR,
6aa8b732
AK		 3136 "kvm",
3137 &kvm_chardev_ops,
3138};
3139
774c47f1
AK		 3140/*
3141 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
3142 * cached on it.
3143 */
3144static void decache_vcpus_on_cpu(int cpu)
3145{
3146 struct kvm *vm;
3147 struct kvm_vcpu *vcpu;
3148 int i;
3149
3150 spin_lock(&kvm_lock);
11ec2804 3151 list_for_each_entry(vm, &vm_list, vm_list)
774c47f1 3152 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
fb3f0f51
RR		 3153 vcpu = vm->vcpus[i];
3154 if (!vcpu)
3155 continue;
774c47f1
AK		 3156 /*
3157 * If the vcpu is locked, then it is running on some
3158 * other cpu and therefore it is not cached on the
3159 * cpu in question.
3160 *
3161 * If it's not locked, check the last cpu it executed
3162 * on.
3163 */
3164 if (mutex_trylock(&vcpu->mutex)) {
3165 if (vcpu->cpu == cpu) {
cbdd1bea 3166 kvm_x86_ops->vcpu_decache(vcpu);
774c47f1
AK		 3167 vcpu->cpu = -1;
3168 }
3169 mutex_unlock(&vcpu->mutex);
3170 }
3171 }
3172 spin_unlock(&kvm_lock);
3173}
3174
1b6c0168
AK		 3175static void hardware_enable(void *junk)
3176{
3177 int cpu = raw_smp_processor_id();
3178
3179 if (cpu_isset(cpu, cpus_hardware_enabled))
3180 return;
3181 cpu_set(cpu, cpus_hardware_enabled);
cbdd1bea 3182 kvm_x86_ops->hardware_enable(NULL);
1b6c0168
AK		 3183}
3184
3185static void hardware_disable(void *junk)
3186{
3187 int cpu = raw_smp_processor_id();
3188
3189 if (!cpu_isset(cpu, cpus_hardware_enabled))
3190 return;
3191 cpu_clear(cpu, cpus_hardware_enabled);
3192 decache_vcpus_on_cpu(cpu);
cbdd1bea 3193 kvm_x86_ops->hardware_disable(NULL);
1b6c0168
AK		 3194}
3195
774c47f1
AK		 3196static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
3197 void *v)
3198{
3199 int cpu = (long)v;
3200
3201 switch (val) {
cec9ad27
AK		 3202 case CPU_DYING:
3203 case CPU_DYING_FROZEN:
6ec8a856
AK		 3204 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
3205 cpu);
3206 hardware_disable(NULL);
3207 break;
774c47f1 3208 case CPU_UP_CANCELED:
8bb78442 3209 case CPU_UP_CANCELED_FROZEN:
43934a38
JK		 3210 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
3211 cpu);
1b6c0168 3212 smp_call_function_single(cpu, hardware_disable, NULL, 0, 1);
774c47f1 3213 break;
43934a38 3214 case CPU_ONLINE:
8bb78442 3215 case CPU_ONLINE_FROZEN:
43934a38
JK		 3216 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
3217 cpu);
1b6c0168 3218 smp_call_function_single(cpu, hardware_enable, NULL, 0, 1);
774c47f1
AK		 3219 break;
3220 }
3221 return NOTIFY_OK;
3222}
3223
9a2b85c6
RR		 3224static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3225 void *v)
3226{
3227 if (val == SYS_RESTART) {
3228 /*
3229 * Some (well, at least mine) BIOSes hang on reboot if
3230 * in vmx root mode.
3231 */
3232 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
3233 on_each_cpu(hardware_disable, NULL, 0, 1);
3234 }
3235 return NOTIFY_OK;
3236}
3237
3238static struct notifier_block kvm_reboot_notifier = {
3239 .notifier_call = kvm_reboot,
3240 .priority = 0,
3241};
3242
2eeb2e94
GH		 3243void kvm_io_bus_init(struct kvm_io_bus *bus)
3244{
3245 memset(bus, 0, sizeof(*bus));
3246}
3247
3248void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3249{
3250 int i;
3251
3252 for (i = 0; i < bus->dev_count; i++) {
3253 struct kvm_io_device *pos = bus->devs[i];
3254
3255 kvm_iodevice_destructor(pos);
3256 }
3257}
3258
3259struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr)
3260{
3261 int i;
3262
3263 for (i = 0; i < bus->dev_count; i++) {
3264 struct kvm_io_device *pos = bus->devs[i];
3265
3266 if (pos->in_range(pos, addr))
3267 return pos;
3268 }
3269
3270 return NULL;
3271}
3272
3273void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
3274{
3275 BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
3276
3277 bus->devs[bus->dev_count++] = dev;
3278}
3279
774c47f1
AK		 3280static struct notifier_block kvm_cpu_notifier = {
3281 .notifier_call = kvm_cpu_hotplug,
3282 .priority = 20, /* must be > scheduler priority */
3283};
3284
1165f5fe
AK		 3285static u64 stat_get(void *_offset)
3286{
3287 unsigned offset = (long)_offset;
3288 u64 total = 0;
3289 struct kvm *kvm;
3290 struct kvm_vcpu *vcpu;
3291 int i;
3292
3293 spin_lock(&kvm_lock);
3294 list_for_each_entry(kvm, &vm_list, vm_list)
3295 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
fb3f0f51
RR		 3296 vcpu = kvm->vcpus[i];
3297 if (vcpu)
3298 total += *(u32 *)((void *)vcpu + offset);
1165f5fe
AK		 3299 }
3300 spin_unlock(&kvm_lock);
3301 return total;
3302}
3303
3dea7ca7 3304DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n");
1165f5fe 3305
6aa8b732
AK		 3306static __init void kvm_init_debug(void)
3307{
3308 struct kvm_stats_debugfs_item *p;
3309
8b6d44c7 3310 debugfs_dir = debugfs_create_dir("kvm", NULL);
6aa8b732 3311 for (p = debugfs_entries; p->name; ++p)
1165f5fe
AK		 3312 p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir,
3313 (void *)(long)p->offset,
3314 &stat_fops);
6aa8b732
AK		 3315}
3316
3317static void kvm_exit_debug(void)
3318{
3319 struct kvm_stats_debugfs_item *p;
3320
3321 for (p = debugfs_entries; p->name; ++p)
3322 debugfs_remove(p->dentry);
3323 debugfs_remove(debugfs_dir);
3324}
3325
59ae6c6b
AK		 3326static int kvm_suspend(struct sys_device *dev, pm_message_t state)
3327{
4267c41a 3328 hardware_disable(NULL);
59ae6c6b
AK		 3329 return 0;
3330}
3331
3332static int kvm_resume(struct sys_device *dev)
3333{
4267c41a 3334 hardware_enable(NULL);
59ae6c6b
AK		 3335 return 0;
3336}
3337
3338static struct sysdev_class kvm_sysdev_class = {
3339 set_kset_name("kvm"),
3340 .suspend = kvm_suspend,
3341 .resume = kvm_resume,
3342};
3343
3344static struct sys_device kvm_sysdev = {
3345 .id = 0,
3346 .cls = &kvm_sysdev_class,
3347};
3348
6aa8b732
AK		 3349hpa_t bad_page_address;
3350
15ad7146
AK		 3351static inline
3352struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3353{
3354 return container_of(pn, struct kvm_vcpu, preempt_notifier);
3355}
3356
3357static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3358{
3359 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3360
cbdd1bea 3361 kvm_x86_ops->vcpu_load(vcpu, cpu);
15ad7146
AK		 3362}
3363
3364static void kvm_sched_out(struct preempt_notifier *pn,
3365 struct task_struct *next)
3366{
3367 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3368
cbdd1bea 3369 kvm_x86_ops->vcpu_put(vcpu);
15ad7146
AK		 3370}
3371
cbdd1bea 3372int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size,
c16f862d 3373 struct module *module)
6aa8b732
AK		 3374{
3375 int r;
002c7f7c 3376 int cpu;
6aa8b732 3377
cbdd1bea 3378 if (kvm_x86_ops) {
09db28b8
YI		 3379 printk(KERN_ERR "kvm: already loaded the other module\n");
3380 return -EEXIST;
3381 }
3382
e097f35c 3383 if (!ops->cpu_has_kvm_support()) {
6aa8b732
AK		 3384 printk(KERN_ERR "kvm: no hardware support\n");
3385 return -EOPNOTSUPP;
3386 }
e097f35c 3387 if (ops->disabled_by_bios()) {
6aa8b732
AK		 3388 printk(KERN_ERR "kvm: disabled by bios\n");
3389 return -EOPNOTSUPP;
3390 }
3391
cbdd1bea 3392 kvm_x86_ops = ops;
e097f35c 3393
cbdd1bea 3394 r = kvm_x86_ops->hardware_setup();
6aa8b732 3395 if (r < 0)
ca45aaae 3396 goto out;
6aa8b732 3397
002c7f7c
YS		 3398 for_each_online_cpu(cpu) {
3399 smp_call_function_single(cpu,
cbdd1bea 3400 kvm_x86_ops->check_processor_compatibility,
002c7f7c
YS		 3401 &r, 0, 1);
3402 if (r < 0)
3403 goto out_free_0;
3404 }
3405
1b6c0168 3406 on_each_cpu(hardware_enable, NULL, 0, 1);
774c47f1
AK		 3407 r = register_cpu_notifier(&kvm_cpu_notifier);
3408 if (r)
3409 goto out_free_1;
6aa8b732
AK		 3410 register_reboot_notifier(&kvm_reboot_notifier);
3411
59ae6c6b
AK		 3412 r = sysdev_class_register(&kvm_sysdev_class);
3413 if (r)
3414 goto out_free_2;
3415
3416 r = sysdev_register(&kvm_sysdev);
3417 if (r)
3418 goto out_free_3;
3419
c16f862d
RR		 3420 /* A kmem cache lets us meet the alignment requirements of fx_save. */
3421 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
3422 __alignof__(struct kvm_vcpu), 0, 0);
3423 if (!kvm_vcpu_cache) {
3424 r = -ENOMEM;
3425 goto out_free_4;
3426 }
3427
6aa8b732
AK		 3428 kvm_chardev_ops.owner = module;
3429
3430 r = misc_register(&kvm_dev);
3431 if (r) {
3432 printk (KERN_ERR "kvm: misc device register failed\n");
3433 goto out_free;
3434 }
3435
15ad7146
AK		 3436 kvm_preempt_ops.sched_in = kvm_sched_in;
3437 kvm_preempt_ops.sched_out = kvm_sched_out;
3438
6aa8b732
AK		 3439 return r;
3440
3441out_free:
c16f862d
RR		 3442 kmem_cache_destroy(kvm_vcpu_cache);
3443out_free_4:
59ae6c6b
AK		 3444 sysdev_unregister(&kvm_sysdev);
3445out_free_3:
3446 sysdev_class_unregister(&kvm_sysdev_class);
3447out_free_2:
6aa8b732 3448 unregister_reboot_notifier(&kvm_reboot_notifier);
774c47f1
AK		 3449 unregister_cpu_notifier(&kvm_cpu_notifier);
3450out_free_1:
1b6c0168 3451 on_each_cpu(hardware_disable, NULL, 0, 1);
002c7f7c 3452out_free_0:
cbdd1bea 3453 kvm_x86_ops->hardware_unsetup();
ca45aaae 3454out:
cbdd1bea 3455 kvm_x86_ops = NULL;
6aa8b732
AK		 3456 return r;
3457}
3458
cbdd1bea 3459void kvm_exit_x86(void)
6aa8b732
AK		 3460{
3461 misc_deregister(&kvm_dev);
c16f862d 3462 kmem_cache_destroy(kvm_vcpu_cache);
59ae6c6b
AK		 3463 sysdev_unregister(&kvm_sysdev);
3464 sysdev_class_unregister(&kvm_sysdev_class);
6aa8b732 3465 unregister_reboot_notifier(&kvm_reboot_notifier);
59ae6c6b 3466 unregister_cpu_notifier(&kvm_cpu_notifier);
1b6c0168 3467 on_each_cpu(hardware_disable, NULL, 0, 1);
cbdd1bea
CE		 3468 kvm_x86_ops->hardware_unsetup();
3469 kvm_x86_ops = NULL;
6aa8b732
AK		 3470}
3471
3472static __init int kvm_init(void)
3473{
3474 static struct page *bad_page;
37e29d90
AK		 3475 int r;
3476
b5a33a75
AK		 3477 r = kvm_mmu_module_init();
3478 if (r)
3479 goto out4;
3480
6aa8b732
AK		 3481 kvm_init_debug();
3482
bf591b24
MR		 3483 kvm_init_msr_list();
3484
6aa8b732
AK		 3485 if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
3486 r = -ENOMEM;
3487 goto out;
3488 }
3489
3490 bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
3491 memset(__va(bad_page_address), 0, PAGE_SIZE);
3492
58e690e6 3493 return 0;
6aa8b732
AK		 3494
3495out:
3496 kvm_exit_debug();
b5a33a75
AK		 3497 kvm_mmu_module_exit();
3498out4:
6aa8b732
AK		 3499 return r;
3500}
3501
3502static __exit void kvm_exit(void)
3503{
3504 kvm_exit_debug();
3505 __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
b5a33a75 3506 kvm_mmu_module_exit();
6aa8b732
AK		 3507}
3508
3509module_init(kvm_init)
3510module_exit(kvm_exit)
3511
cbdd1bea
CE		 3512EXPORT_SYMBOL_GPL(kvm_init_x86);
3513EXPORT_SYMBOL_GPL(kvm_exit_x86);