[GitHub/mt8127/android_kernel_alcatel_ttab.git] / arch / x86 / kernel / paravirt.c

/*  Paravirtualization interfaces
    Copyright (C) 2006 Rusty Russell IBM Corporation

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

    2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
*/

#include <linux/errno.h>
#include <linux/module.h>
#include <linux/efi.h>
#include <linux/bcd.h>
#include <linux/highmem.h>

#include <asm/bug.h>
#include <asm/paravirt.h>
#include <asm/debugreg.h>
#include <asm/desc.h>
#include <asm/setup.h>
#include <asm/pgtable.h>
#include <asm/time.h>
#include <asm/pgalloc.h>
#include <asm/irq.h>
#include <asm/delay.h>
#include <asm/fixmap.h>
#include <asm/apic.h>
#include <asm/tlbflush.h>
#include <asm/timer.h>
#include <asm/special_insns.h>

/* nop stub */
void _paravirt_nop(void)
{
}

/* identity function, which can be inlined */
u32 _paravirt_ident_32(u32 x)
{
	return x;
}

u64 _paravirt_ident_64(u64 x)
{
	return x;
}

void __init default_banner(void)
{
	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
	       pv_info.name);
}

/* Simple instruction patching code. */
#define DEF_NATIVE(ops, name, code)					\
	extern const char start_##ops##_##name[], end_##ops##_##name[];	\
	asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")

/* Undefined instruction for dealing with missing ops pointers. */
static const unsigned char ud2a[] = { 0x0f, 0x0b };

unsigned paravirt_patch_nop(void)
{
	return 0;
}

unsigned paravirt_patch_ignore(unsigned len)
{
	return len;
}

struct branch {
	unsigned char opcode;
	u32 delta;
} __attribute__((packed));

unsigned paravirt_patch_call(void *insnbuf,
			     const void *target, u16 tgt_clobbers,
			     unsigned long addr, u16 site_clobbers,
			     unsigned len)
{
	struct branch *b = insnbuf;
	unsigned long delta = (unsigned long)target - (addr+5);

	if (tgt_clobbers & ~site_clobbers)
		return len;	/* target would clobber too much for this site */
	if (len < 5)
		return len;	/* call too long for patch site */

	b->opcode = 0xe8; /* call */
	b->delta = delta;
	BUILD_BUG_ON(sizeof(*b) != 5);

	return 5;
}

unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
			    unsigned long addr, unsigned len)
{
	struct branch *b = insnbuf;
	unsigned long delta = (unsigned long)target - (addr+5);

	if (len < 5)
		return len;	/* call too long for patch site */

	b->opcode = 0xe9;	/* jmp */
	b->delta = delta;

	return 5;
}

/* Neat trick to map patch type back to the call within the
 * corresponding structure. */
static void *get_call_destination(u8 type)
{
	struct paravirt_patch_template tmpl = {
		.pv_init_ops = pv_init_ops,
		.pv_time_ops = pv_time_ops,
		.pv_cpu_ops = pv_cpu_ops,
		.pv_irq_ops = pv_irq_ops,
		.pv_apic_ops = pv_apic_ops,
		.pv_mmu_ops = pv_mmu_ops,
#ifdef CONFIG_PARAVIRT_SPINLOCKS
		.pv_lock_ops = pv_lock_ops,
#endif
	};
	return *((void **)&tmpl + type);
}

unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
				unsigned long addr, unsigned len)
{
	void *opfunc = get_call_destination(type);
	unsigned ret;

	if (opfunc == NULL)
		/* If there's no function, patch it with a ud2a (BUG) */
		ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
	else if (opfunc == _paravirt_nop)
		/* If the operation is a nop, then nop the callsite */
		ret = paravirt_patch_nop();

	/* identity functions just return their single argument */
	else if (opfunc == _paravirt_ident_32)
		ret = paravirt_patch_ident_32(insnbuf, len);
	else if (opfunc == _paravirt_ident_64)
		ret = paravirt_patch_ident_64(insnbuf, len);

	else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||
		 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) ||
		 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64))
		/* If operation requires a jmp, then jmp */
		ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
	else
		/* Otherwise call the function; assume target could
		   clobber any caller-save reg */
		ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY,
					  addr, clobbers, len);

	return ret;
}

unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
			      const char *start, const char *end)
{
	unsigned insn_len = end - start;

	if (insn_len > len || start == NULL)
		insn_len = len;
	else
		memcpy(insnbuf, start, insn_len);

	return insn_len;
}

static void native_flush_tlb(void)
{
	__native_flush_tlb();
}

/*
 * Global pages have to be flushed a bit differently. Not a real
 * performance problem because this does not happen often.
 */
static void native_flush_tlb_global(void)
{
	__native_flush_tlb_global();
}

static void native_flush_tlb_single(unsigned long addr)
{
	__native_flush_tlb_single(addr);
}

struct static_key paravirt_steal_enabled;
struct static_key paravirt_steal_rq_enabled;

static u64 native_steal_clock(int cpu)
{
	return 0;
}

/* These are in entry.S */
extern void native_iret(void);
extern void native_irq_enable_sysexit(void);
extern void native_usergs_sysret32(void);
extern void native_usergs_sysret64(void);

static struct resource reserve_ioports = {
	.start = 0,
	.end = IO_SPACE_LIMIT,
	.name = "paravirt-ioport",
	.flags = IORESOURCE_IO | IORESOURCE_BUSY,
};

/*
 * Reserve the whole legacy IO space to prevent any legacy drivers
 * from wasting time probing for their hardware.  This is a fairly
 * brute-force approach to disabling all non-virtual drivers.
 *
 * Note that this must be called very early to have any effect.
 */
int paravirt_disable_iospace(void)
{
	return request_resource(&ioport_resource, &reserve_ioports);
}

static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;

static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
	BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);

	this_cpu_write(paravirt_lazy_mode, mode);
}

static void leave_lazy(enum paravirt_lazy_mode mode)
{
	BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);

	this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}

void paravirt_enter_lazy_mmu(void)
{
	enter_lazy(PARAVIRT_LAZY_MMU);
}

void paravirt_leave_lazy_mmu(void)
{
	leave_lazy(PARAVIRT_LAZY_MMU);
}

void paravirt_flush_lazy_mmu(void)
{
	preempt_disable();

	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
		arch_leave_lazy_mmu_mode();
		arch_enter_lazy_mmu_mode();
	}

	preempt_enable();
}

void paravirt_start_context_switch(struct task_struct *prev)
{
	BUG_ON(preemptible());

	if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
		arch_leave_lazy_mmu_mode();
		set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
	}
	enter_lazy(PARAVIRT_LAZY_CPU);
}

void paravirt_end_context_switch(struct task_struct *next)
{
	BUG_ON(preemptible());

	leave_lazy(PARAVIRT_LAZY_CPU);

	if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
		arch_enter_lazy_mmu_mode();
}

enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
{
	if (in_interrupt())
		return PARAVIRT_LAZY_NONE;

	return this_cpu_read(paravirt_lazy_mode);
}

struct pv_info pv_info = {
	.name = "bare hardware",
	.paravirt_enabled = 0,
	.kernel_rpl = 0,
	.shared_kernel_pmd = 1,	/* Only used when CONFIG_X86_PAE is set */

#ifdef CONFIG_X86_64
	.extra_user_64bit_cs = __USER_CS,
#endif
};

struct pv_init_ops pv_init_ops = {
	.patch = native_patch,
};

struct pv_time_ops pv_time_ops = {
	.sched_clock = native_sched_clock,
	.steal_clock = native_steal_clock,
};

struct pv_irq_ops pv_irq_ops = {
	.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
	.restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
	.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
	.irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
	.safe_halt = native_safe_halt,
	.halt = native_halt,
#ifdef CONFIG_X86_64
	.adjust_exception_frame = paravirt_nop,
#endif
};

struct pv_cpu_ops pv_cpu_ops = {
	.cpuid = native_cpuid,
	.get_debugreg = native_get_debugreg,
	.set_debugreg = native_set_debugreg,
	.clts = native_clts,
	.read_cr0 = native_read_cr0,
	.write_cr0 = native_write_cr0,
	.read_cr4 = native_read_cr4,
	.read_cr4_safe = native_read_cr4_safe,
	.write_cr4 = native_write_cr4,
#ifdef CONFIG_X86_64
	.read_cr8 = native_read_cr8,
	.write_cr8 = native_write_cr8,
#endif
	.wbinvd = native_wbinvd,
	.read_msr = native_read_msr_safe,
	.write_msr = native_write_msr_safe,
	.read_tsc = native_read_tsc,
	.read_pmc = native_read_pmc,
	.read_tscp = native_read_tscp,
	.load_tr_desc = native_load_tr_desc,
	.set_ldt = native_set_ldt,
	.load_gdt = native_load_gdt,
	.load_idt = native_load_idt,
	.store_gdt = native_store_gdt,
	.store_idt = native_store_idt,
	.store_tr = native_store_tr,
	.load_tls = native_load_tls,
#ifdef CONFIG_X86_64
	.load_gs_index = native_load_gs_index,
#endif
	.write_ldt_entry = native_write_ldt_entry,
	.write_gdt_entry = native_write_gdt_entry,
	.write_idt_entry = native_write_idt_entry,

	.alloc_ldt = paravirt_nop,
	.free_ldt = paravirt_nop,

	.load_sp0 = native_load_sp0,

#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
	.irq_enable_sysexit = native_irq_enable_sysexit,
#endif
#ifdef CONFIG_X86_64
#ifdef CONFIG_IA32_EMULATION
	.usergs_sysret32 = native_usergs_sysret32,
#endif
	.usergs_sysret64 = native_usergs_sysret64,
#endif
	.iret = native_iret,
	.swapgs = native_swapgs,

	.set_iopl_mask = native_set_iopl_mask,
	.io_delay = native_io_delay,

	.start_context_switch = paravirt_nop,
	.end_context_switch = paravirt_nop,
};

struct pv_apic_ops pv_apic_ops = {
#ifdef CONFIG_X86_LOCAL_APIC
	.startup_ipi_hook = paravirt_nop,
#endif
};

#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
/* 32-bit pagetable entries */
#define PTE_IDENT	__PV_IS_CALLEE_SAVE(_paravirt_ident_32)
#else
/* 64-bit pagetable entries */
#define PTE_IDENT	__PV_IS_CALLEE_SAVE(_paravirt_ident_64)
#endif

struct pv_mmu_ops pv_mmu_ops = {

	.read_cr2 = native_read_cr2,
	.write_cr2 = native_write_cr2,
	.read_cr3 = native_read_cr3,
	.write_cr3 = native_write_cr3,

	.flush_tlb_user = native_flush_tlb,
	.flush_tlb_kernel = native_flush_tlb_global,
	.flush_tlb_single = native_flush_tlb_single,
	.flush_tlb_others = native_flush_tlb_others,

	.pgd_alloc = __paravirt_pgd_alloc,
	.pgd_free = paravirt_nop,

	.alloc_pte = paravirt_nop,
	.alloc_pmd = paravirt_nop,
	.alloc_pud = paravirt_nop,
	.release_pte = paravirt_nop,
	.release_pmd = paravirt_nop,
	.release_pud = paravirt_nop,

	.set_pte = native_set_pte,
	.set_pte_at = native_set_pte_at,
	.set_pmd = native_set_pmd,
	.set_pmd_at = native_set_pmd_at,
	.pte_update = paravirt_nop,
	.pte_update_defer = paravirt_nop,
	.pmd_update = paravirt_nop,
	.pmd_update_defer = paravirt_nop,

	.ptep_modify_prot_start = __ptep_modify_prot_start,
	.ptep_modify_prot_commit = __ptep_modify_prot_commit,

#if PAGETABLE_LEVELS >= 3
#ifdef CONFIG_X86_PAE
	.set_pte_atomic = native_set_pte_atomic,
	.pte_clear = native_pte_clear,
	.pmd_clear = native_pmd_clear,
#endif
	.set_pud = native_set_pud,

	.pmd_val = PTE_IDENT,
	.make_pmd = PTE_IDENT,

#if PAGETABLE_LEVELS == 4
	.pud_val = PTE_IDENT,
	.make_pud = PTE_IDENT,

	.set_pgd = native_set_pgd,
#endif
#endif /* PAGETABLE_LEVELS >= 3 */

	.pte_val = PTE_IDENT,
	.pgd_val = PTE_IDENT,

	.make_pte = PTE_IDENT,
	.make_pgd = PTE_IDENT,

	.dup_mmap = paravirt_nop,
	.exit_mmap = paravirt_nop,
	.activate_mm = paravirt_nop,

	.lazy_mode = {
		.enter = paravirt_nop,
		.leave = paravirt_nop,
		.flush = paravirt_nop,
	},

	.set_fixmap = native_set_fixmap,
};

EXPORT_SYMBOL_GPL(pv_time_ops);
EXPORT_SYMBOL    (pv_cpu_ops);
EXPORT_SYMBOL    (pv_mmu_ops);
EXPORT_SYMBOL_GPL(pv_apic_ops);
EXPORT_SYMBOL_GPL(pv_info);
EXPORT_SYMBOL    (pv_irq_ops);
Commit	Line	Data
d3561b7f RR	1	/* Paravirtualization interfaces
	2	Copyright (C) 2006 Rusty Russell IBM Corporation
	3
	4	This program is free software; you can redistribute it and/or modify
	5	it under the terms of the GNU General Public License as published by
	6	the Free Software Foundation; either version 2 of the License, or
	7	(at your option) any later version.
	8
	9	This program is distributed in the hope that it will be useful,
	10	but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	GNU General Public License for more details.
	13
	14	You should have received a copy of the GNU General Public License
	15	along with this program; if not, write to the Free Software
	16	Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
b1df07bd GOC	17
b1df07bd GOC	18	2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
d3561b7f	19	*/
b1df07bd	20
d3561b7f RR	21	#include <linux/errno.h>
	22	#include <linux/module.h>
	23	#include <linux/efi.h>
	24	#include <linux/bcd.h>
ce6234b5	25	#include <linux/highmem.h>
d3561b7f RR	26
	27	#include <asm/bug.h>
	28	#include <asm/paravirt.h>
50af5ead	29	#include <asm/debugreg.h>
d3561b7f RR	30	#include <asm/desc.h>
d3561b7f RR	31	#include <asm/setup.h>
a312b37b	32	#include <asm/pgtable.h>
d3561b7f	33	#include <asm/time.h>
eba0045f	34	#include <asm/pgalloc.h>
d3561b7f RR	35	#include <asm/irq.h>
d3561b7f RR	36	#include <asm/delay.h>
13623d79 RR	37	#include <asm/fixmap.h>
13623d79 RR	38	#include <asm/apic.h>
da181a8b	39	#include <asm/tlbflush.h>
6cb9a835	40	#include <asm/timer.h>
f05e798a	41	#include <asm/special_insns.h>
d3561b7f RR	42
d3561b7f RR	43	/* nop stub */
45876233	44	void _paravirt_nop(void)
d3561b7f RR	45	{
	46	}
	47
41edafdb JF	48	/* identity function, which can be inlined */
	49	u32 _paravirt_ident_32(u32 x)
	50	{
	51	return x;
	52	}
	53
	54	u64 _paravirt_ident_64(u64 x)
	55	{
	56	return x;
	57	}
	58
6f30c1ac	59	void __init default_banner(void)
d3561b7f RR	60	{
d3561b7f RR	61	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
93b1eab3	62	pv_info.name);
d3561b7f RR	63	}
d3561b7f RR	64
139ec7c4	65	/* Simple instruction patching code. */
93b1eab3 JF	66	#define DEF_NATIVE(ops, name, code) \
	67	extern const char start_##ops##_##name[], end_##ops##_##name[]; \
	68	asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
	69
93b1eab3 JF	70	/* Undefined instruction for dealing with missing ops pointers. */
93b1eab3 JF	71	static const unsigned char ud2a[] = { 0x0f, 0x0b };
139ec7c4	72
63f70270 JF	73	unsigned paravirt_patch_nop(void)
	74	{
	75	return 0;
	76	}
	77
	78	unsigned paravirt_patch_ignore(unsigned len)
	79	{
	80	return len;
	81	}
	82
19d36ccd AK	83	struct branch {
	84	unsigned char opcode;
	85	u32 delta;
	86	} __attribute__((packed));
	87
ab144f5e AK	88	unsigned paravirt_patch_call(void *insnbuf,
	89	const void *target, u16 tgt_clobbers,
	90	unsigned long addr, u16 site_clobbers,
63f70270 JF	91	unsigned len)
63f70270 JF	92	{
ab144f5e AK	93	struct branch *b = insnbuf;
ab144f5e AK	94	unsigned long delta = (unsigned long)target - (addr+5);
63f70270 JF	95
	96	if (tgt_clobbers & ~site_clobbers)
	97	return len; /* target would clobber too much for this site */
	98	if (len < 5)
	99	return len; /* call too long for patch site */
139ec7c4	100
ab144f5e AK	101	b->opcode = 0xe8; /* call */
	102	b->delta = delta;
	103	BUILD_BUG_ON(sizeof(*b) != 5);
139ec7c4	104
63f70270 JF	105	return 5;
	106	}
	107
93b1eab3	108	unsigned paravirt_patch_jmp(void insnbuf, const void target,
ab144f5e	109	unsigned long addr, unsigned len)
63f70270	110	{
ab144f5e AK	111	struct branch *b = insnbuf;
ab144f5e AK	112	unsigned long delta = (unsigned long)target - (addr+5);
63f70270 JF	113
	114	if (len < 5)
	115	return len; /* call too long for patch site */
	116
ab144f5e AK	117	b->opcode = 0xe9; /* jmp */
ab144f5e AK	118	b->delta = delta;
63f70270 JF	119
	120	return 5;
	121	}
	122
93b1eab3 JF	123	/* Neat trick to map patch type back to the call within the
	124	* corresponding structure. */
	125	static void *get_call_destination(u8 type)
	126	{
	127	struct paravirt_patch_template tmpl = {
	128	.pv_init_ops = pv_init_ops,
93b1eab3 JF	129	.pv_time_ops = pv_time_ops,
	130	.pv_cpu_ops = pv_cpu_ops,
	131	.pv_irq_ops = pv_irq_ops,
	132	.pv_apic_ops = pv_apic_ops,
	133	.pv_mmu_ops = pv_mmu_ops,
b4ecc126	134	#ifdef CONFIG_PARAVIRT_SPINLOCKS
74d4affd	135	.pv_lock_ops = pv_lock_ops,
b4ecc126	136	#endif
93b1eab3 JF	137	};
	138	return ((void *)&tmpl + type);
	139	}
	140
ab144f5e AK	141	unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
ab144f5e AK	142	unsigned long addr, unsigned len)
63f70270	143	{
93b1eab3	144	void *opfunc = get_call_destination(type);
63f70270 JF	145	unsigned ret;
	146
	147	if (opfunc == NULL)
	148	/* If there's no function, patch it with a ud2a (BUG) */
93b1eab3	149	ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a));
41edafdb	150	else if (opfunc == _paravirt_nop)
63f70270 JF	151	/* If the operation is a nop, then nop the callsite */
63f70270 JF	152	ret = paravirt_patch_nop();
41edafdb JF	153
	154	/* identity functions just return their single argument */
	155	else if (opfunc == _paravirt_ident_32)
	156	ret = paravirt_patch_ident_32(insnbuf, len);
	157	else if (opfunc == _paravirt_ident_64)
	158	ret = paravirt_patch_ident_64(insnbuf, len);
	159
93b1eab3	160	else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) \|\|
d75cd22f	161	type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) \|\|
2be29982 JF	162	type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret32) \|\|
2be29982 JF	163	type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64))
63f70270	164	/* If operation requires a jmp, then jmp */
93b1eab3	165	ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
63f70270 JF	166	else
	167	/* Otherwise call the function; assume target could
	168	clobber any caller-save reg */
ab144f5e AK	169	ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY,
ab144f5e AK	170	addr, clobbers, len);
63f70270 JF	171
	172	return ret;
	173	}
	174
ab144f5e	175	unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
63f70270 JF	176	const char start, const char end)
	177	{
	178	unsigned insn_len = end - start;
139ec7c4	179
63f70270 JF	180	if (insn_len > len \|\| start == NULL)
	181	insn_len = len;
	182	else
ab144f5e	183	memcpy(insnbuf, start, insn_len);
139ec7c4	184
139ec7c4 RR	185	return insn_len;
	186	}
	187
1a1eecd1	188	static void native_flush_tlb(void)
da181a8b RR	189	{
	190	__native_flush_tlb();
	191	}
	192
	193	/*
	194	* Global pages have to be flushed a bit differently. Not a real
	195	* performance problem because this does not happen often.
	196	*/
1a1eecd1	197	static void native_flush_tlb_global(void)
da181a8b RR	198	{
	199	__native_flush_tlb_global();
	200	}
	201
63f70270	202	static void native_flush_tlb_single(unsigned long addr)
da181a8b RR	203	{
	204	__native_flush_tlb_single(addr);
	205	}
	206
c5905afb IM	207	struct static_key paravirt_steal_enabled;
c5905afb IM	208	struct static_key paravirt_steal_rq_enabled;
3c404b57 GC	209
	210	static u64 native_steal_clock(int cpu)
	211	{
	212	return 0;
	213	}
	214
d3561b7f	215	/* These are in entry.S */
1a1eecd1	216	extern void native_iret(void);
d75cd22f	217	extern void native_irq_enable_sysexit(void);
2be29982 JF	218	extern void native_usergs_sysret32(void);
2be29982 JF	219	extern void native_usergs_sysret64(void);
d3561b7f	220
d572929c JF	221	static struct resource reserve_ioports = {
	222	.start = 0,
	223	.end = IO_SPACE_LIMIT,
	224	.name = "paravirt-ioport",
	225	.flags = IORESOURCE_IO \| IORESOURCE_BUSY,
	226	};
	227
d572929c JF	228	/*
	229	* Reserve the whole legacy IO space to prevent any legacy drivers
	230	* from wasting time probing for their hardware. This is a fairly
	231	* brute-force approach to disabling all non-virtual drivers.
	232	*
	233	* Note that this must be called very early to have any effect.
	234	*/
	235	int paravirt_disable_iospace(void)
	236	{
f7743fe6	237	return request_resource(&ioport_resource, &reserve_ioports);
d572929c JF	238	}
d572929c JF	239
8965c1c0 JF	240	static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
	241
	242	static inline void enter_lazy(enum paravirt_lazy_mode mode)
	243	{
c6ae41e7	244	BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
8965c1c0	245
c6ae41e7	246	this_cpu_write(paravirt_lazy_mode, mode);
8965c1c0 JF	247	}
8965c1c0 JF	248
b407fc57	249	static void leave_lazy(enum paravirt_lazy_mode mode)
8965c1c0	250	{
c6ae41e7	251	BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);
8965c1c0	252
c6ae41e7	253	this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
8965c1c0 JF	254	}
	255
	256	void paravirt_enter_lazy_mmu(void)
	257	{
	258	enter_lazy(PARAVIRT_LAZY_MMU);
	259	}
	260
	261	void paravirt_leave_lazy_mmu(void)
	262	{
b407fc57	263	leave_lazy(PARAVIRT_LAZY_MMU);
8965c1c0 JF	264	}
8965c1c0 JF	265
511ba86e BO	266	void paravirt_flush_lazy_mmu(void)
	267	{
	268	preempt_disable();
	269
	270	if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
	271	arch_leave_lazy_mmu_mode();
	272	arch_enter_lazy_mmu_mode();
	273	}
	274
	275	preempt_enable();
	276	}
	277
224101ed	278	void paravirt_start_context_switch(struct task_struct *prev)
8965c1c0	279	{
2829b449 JF	280	BUG_ON(preemptible());
2829b449 JF	281
c6ae41e7	282	if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
b407fc57	283	arch_leave_lazy_mmu_mode();
224101ed	284	set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
b407fc57	285	}
8965c1c0 JF	286	enter_lazy(PARAVIRT_LAZY_CPU);
	287	}
	288
224101ed	289	void paravirt_end_context_switch(struct task_struct *next)
8965c1c0	290	{
2829b449 JF	291	BUG_ON(preemptible());
2829b449 JF	292
b407fc57 JF	293	leave_lazy(PARAVIRT_LAZY_CPU);
b407fc57 JF	294
224101ed	295	if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
b407fc57	296	arch_enter_lazy_mmu_mode();
8965c1c0 JF	297	}
	298
	299	enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
	300	{
b8bcfe99 JF	301	if (in_interrupt())
	302	return PARAVIRT_LAZY_NONE;
	303
c6ae41e7	304	return this_cpu_read(paravirt_lazy_mode);
8965c1c0 JF	305	}
8965c1c0 JF	306
93b1eab3	307	struct pv_info pv_info = {
d3561b7f RR	308	.name = "bare hardware",
	309	.paravirt_enabled = 0,
	310	.kernel_rpl = 0,
5311ab62	311	.shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */
318f5a2a AL	312
	313	#ifdef CONFIG_X86_64
	314	.extra_user_64bit_cs = __USER_CS,
	315	#endif
93b1eab3	316	};
d3561b7f	317
93b1eab3 JF	318	struct pv_init_ops pv_init_ops = {
93b1eab3 JF	319	.patch = native_patch,
93b1eab3 JF	320	};
	321
	322	struct pv_time_ops pv_time_ops = {
93b1eab3	323	.sched_clock = native_sched_clock,
3c404b57	324	.steal_clock = native_steal_clock,
93b1eab3 JF	325	};
	326
	327	struct pv_irq_ops pv_irq_ops = {
ecb93d1c JF	328	.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
	329	.restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
	330	.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
	331	.irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
93b1eab3 JF	332	.safe_halt = native_safe_halt,
93b1eab3 JF	333	.halt = native_halt,
fab58420 JF	334	#ifdef CONFIG_X86_64
	335	.adjust_exception_frame = paravirt_nop,
	336	#endif
93b1eab3	337	};
d3561b7f	338
93b1eab3	339	struct pv_cpu_ops pv_cpu_ops = {
d3561b7f RR	340	.cpuid = native_cpuid,
	341	.get_debugreg = native_get_debugreg,
	342	.set_debugreg = native_set_debugreg,
	343	.clts = native_clts,
	344	.read_cr0 = native_read_cr0,
	345	.write_cr0 = native_write_cr0,
d3561b7f RR	346	.read_cr4 = native_read_cr4,
	347	.read_cr4_safe = native_read_cr4_safe,
	348	.write_cr4 = native_write_cr4,
88b4755f GOC	349	#ifdef CONFIG_X86_64
	350	.read_cr8 = native_read_cr8,
	351	.write_cr8 = native_write_cr8,
	352	#endif
d3561b7f	353	.wbinvd = native_wbinvd,
90a0a06a RR	354	.read_msr = native_read_msr_safe,
90a0a06a RR	355	.write_msr = native_write_msr_safe,
d3561b7f RR	356	.read_tsc = native_read_tsc,
d3561b7f RR	357	.read_pmc = native_read_pmc,
e5aaac44	358	.read_tscp = native_read_tscp,
d3561b7f RR	359	.load_tr_desc = native_load_tr_desc,
	360	.set_ldt = native_set_ldt,
	361	.load_gdt = native_load_gdt,
	362	.load_idt = native_load_idt,
	363	.store_gdt = native_store_gdt,
	364	.store_idt = native_store_idt,
	365	.store_tr = native_store_tr,
	366	.load_tls = native_load_tls,
9f9d489a JF	367	#ifdef CONFIG_X86_64
	368	.load_gs_index = native_load_gs_index,
	369	#endif
75b8bb3e	370	.write_ldt_entry = native_write_ldt_entry,
014b15be	371	.write_gdt_entry = native_write_gdt_entry,
8d947344	372	.write_idt_entry = native_write_idt_entry,
38ffbe66 JF	373
	374	.alloc_ldt = paravirt_nop,
	375	.free_ldt = paravirt_nop,
	376
faca6227	377	.load_sp0 = native_load_sp0,
d3561b7f	378
102d0a4b	379	#if defined(CONFIG_X86_32) \|\| defined(CONFIG_IA32_EMULATION)
d75cd22f	380	.irq_enable_sysexit = native_irq_enable_sysexit,
102d0a4b	381	#endif
2be29982	382	#ifdef CONFIG_X86_64
102d0a4b	383	#ifdef CONFIG_IA32_EMULATION
2be29982	384	.usergs_sysret32 = native_usergs_sysret32,
102d0a4b	385	#endif
2be29982	386	.usergs_sysret64 = native_usergs_sysret64,
d75cd22f	387	#endif
93b1eab3	388	.iret = native_iret,
e801f864	389	.swapgs = native_swapgs,
93b1eab3	390
d3561b7f RR	391	.set_iopl_mask = native_set_iopl_mask,
d3561b7f RR	392	.io_delay = native_io_delay,
8965c1c0	393
224101ed JF	394	.start_context_switch = paravirt_nop,
224101ed JF	395	.end_context_switch = paravirt_nop,
93b1eab3	396	};
d3561b7f	397
93b1eab3	398	struct pv_apic_ops pv_apic_ops = {
13623d79	399	#ifdef CONFIG_X86_LOCAL_APIC
0260c196	400	.startup_ipi_hook = paravirt_nop,
13623d79	401	#endif
93b1eab3 JF	402	};
93b1eab3 JF	403
41edafdb JF	404	#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
41edafdb JF	405	/* 32-bit pagetable entries */
da5de7c2	406	#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
41edafdb JF	407	#else
41edafdb JF	408	/* 64-bit pagetable entries */
da5de7c2	409	#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
41edafdb JF	410	#endif
41edafdb JF	411
93b1eab3	412	struct pv_mmu_ops pv_mmu_ops = {
b239fb25	413
93b1eab3 JF	414	.read_cr2 = native_read_cr2,
	415	.write_cr2 = native_write_cr2,
	416	.read_cr3 = native_read_cr3,
	417	.write_cr3 = native_write_cr3,
	418
da181a8b RR	419	.flush_tlb_user = native_flush_tlb,
	420	.flush_tlb_kernel = native_flush_tlb_global,
	421	.flush_tlb_single = native_flush_tlb_single,
d4c10477	422	.flush_tlb_others = native_flush_tlb_others,
da181a8b	423
eba0045f JF	424	.pgd_alloc = __paravirt_pgd_alloc,
	425	.pgd_free = paravirt_nop,
	426
6944a9c8 JF	427	.alloc_pte = paravirt_nop,
6944a9c8 JF	428	.alloc_pmd = paravirt_nop,
2761fa09	429	.alloc_pud = paravirt_nop,
6944a9c8 JF	430	.release_pte = paravirt_nop,
6944a9c8 JF	431	.release_pmd = paravirt_nop,
2761fa09	432	.release_pud = paravirt_nop,
c119ecce	433
da181a8b RR	434	.set_pte = native_set_pte,
	435	.set_pte_at = native_set_pte_at,
	436	.set_pmd = native_set_pmd,
331127f7	437	.set_pmd_at = native_set_pmd_at,
45876233 JF	438	.pte_update = paravirt_nop,
45876233 JF	439	.pte_update_defer = paravirt_nop,
331127f7 AA	440	.pmd_update = paravirt_nop,
331127f7 AA	441	.pmd_update_defer = paravirt_nop,
3dc494e8	442
08b882c6 JF	443	.ptep_modify_prot_start = __ptep_modify_prot_start,
	444	.ptep_modify_prot_commit = __ptep_modify_prot_commit,
	445
f95f2f7b	446	#if PAGETABLE_LEVELS >= 3
da181a8b RR	447	#ifdef CONFIG_X86_PAE
da181a8b RR	448	.set_pte_atomic = native_set_pte_atomic,
da181a8b RR	449	.pte_clear = native_pte_clear,
da181a8b RR	450	.pmd_clear = native_pmd_clear,
f95f2f7b EH	451	#endif
f95f2f7b EH	452	.set_pud = native_set_pud,
da5de7c2 JF	453
	454	.pmd_val = PTE_IDENT,
	455	.make_pmd = PTE_IDENT,
f95f2f7b EH	456
f95f2f7b EH	457	#if PAGETABLE_LEVELS == 4
da5de7c2 JF	458	.pud_val = PTE_IDENT,
	459	.make_pud = PTE_IDENT,
	460
f95f2f7b	461	.set_pgd = native_set_pgd,
da181a8b	462	#endif
f95f2f7b	463	#endif /* PAGETABLE_LEVELS >= 3 */
da181a8b	464
da5de7c2 JF	465	.pte_val = PTE_IDENT,
da5de7c2 JF	466	.pgd_val = PTE_IDENT,
3dc494e8	467
da5de7c2 JF	468	.make_pte = PTE_IDENT,
da5de7c2 JF	469	.make_pgd = PTE_IDENT,
3dc494e8	470
d6dd61c8 JF	471	.dup_mmap = paravirt_nop,
	472	.exit_mmap = paravirt_nop,
	473	.activate_mm = paravirt_nop,
8965c1c0 JF	474
	475	.lazy_mode = {
	476	.enter = paravirt_nop,
	477	.leave = paravirt_nop,
511ba86e	478	.flush = paravirt_nop,
8965c1c0	479	},
aeaaa59c JF	480
aeaaa59c JF	481	.set_fixmap = native_set_fixmap,
d3561b7f	482	};
0dbe5a11	483
93b1eab3	484	EXPORT_SYMBOL_GPL(pv_time_ops);
f97b8954 JF	485	EXPORT_SYMBOL (pv_cpu_ops);
f97b8954 JF	486	EXPORT_SYMBOL (pv_mmu_ops);
93b1eab3 JF	487	EXPORT_SYMBOL_GPL(pv_apic_ops);
	488	EXPORT_SYMBOL_GPL(pv_info);
	489	EXPORT_SYMBOL (pv_irq_ops);