[GitHub/mt8127/android_kernel_alcatel_ttab.git] / include / asm-i386 / system.h

#ifndef __ASM_SYSTEM_H
#define __ASM_SYSTEM_H

#include <linux/kernel.h>
#include <asm/segment.h>
#include <asm/cpufeature.h>
#include <linux/bitops.h> /* for LOCK_PREFIX */

#ifdef __KERNEL__

struct task_struct;	/* one of the stranger aspects of C forward declarations.. */
extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next));

/*
 * Saving eflags is important. It switches not only IOPL between tasks,
 * it also protects other tasks from NT leaking through sysenter etc.
 */
#define switch_to(prev,next,last) do {					\
	unsigned long esi,edi;						\
	asm volatile("pushfl\n\t"		/* Save flags */	\
		     "pushl %%ebp\n\t"					\
		     "movl %%esp,%0\n\t"	/* save ESP */		\
		     "movl %5,%%esp\n\t"	/* restore ESP */	\
		     "movl $1f,%1\n\t"		/* save EIP */		\
		     "pushl %6\n\t"		/* restore EIP */	\
		     "jmp __switch_to\n"				\
		     "1:\t"						\
		     "popl %%ebp\n\t"					\
		     "popfl"						\
		     :"=m" (prev->thread.esp),"=m" (prev->thread.eip),	\
		      "=a" (last),"=S" (esi),"=D" (edi)			\
		     :"m" (next->thread.esp),"m" (next->thread.eip),	\
		      "2" (prev), "d" (next));				\
} while (0)

#define _set_base(addr,base) do { unsigned long __pr; \
__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	"rorl $16,%%edx\n\t" \
	"movb %%dl,%2\n\t" \
	"movb %%dh,%3" \
	:"=&d" (__pr) \
	:"m" (*((addr)+2)), \
	 "m" (*((addr)+4)), \
	 "m" (*((addr)+7)), \
         "0" (base) \
        ); } while(0)

#define _set_limit(addr,limit) do { unsigned long __lr; \
__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	"rorl $16,%%edx\n\t" \
	"movb %2,%%dh\n\t" \
	"andb $0xf0,%%dh\n\t" \
	"orb %%dh,%%dl\n\t" \
	"movb %%dl,%2" \
	:"=&d" (__lr) \
	:"m" (*(addr)), \
	 "m" (*((addr)+6)), \
	 "0" (limit) \
        ); } while(0)

#define set_base(ldt,base) _set_base( ((char *)&(ldt)) , (base) )
#define set_limit(ldt,limit) _set_limit( ((char *)&(ldt)) , ((limit)-1) )

/*
 * Load a segment. Fall back on loading the zero
 * segment if something goes wrong..
 */
#define loadsegment(seg,value)			\
	asm volatile("\n"			\
		"1:\t"				\
		"mov %0,%%" #seg "\n"		\
		"2:\n"				\
		".section .fixup,\"ax\"\n"	\
		"3:\t"				\
		"pushl $0\n\t"			\
		"popl %%" #seg "\n\t"		\
		"jmp 2b\n"			\
		".previous\n"			\
		".section __ex_table,\"a\"\n\t"	\
		".align 4\n\t"			\
		".long 1b,3b\n"			\
		".previous"			\
		: :"rm" (value))

/*
 * Save a segment register away
 */
#define savesegment(seg, value) \
	asm volatile("mov %%" #seg ",%0":"=rm" (value))


static inline void native_clts(void)
{
	asm volatile ("clts");
}

static inline unsigned long native_read_cr0(void)
{
	unsigned long val;
	asm volatile("movl %%cr0,%0\n\t" :"=r" (val));
	return val;
}

static inline void native_write_cr0(unsigned long val)
{
	asm volatile("movl %0,%%cr0": :"r" (val));
}

static inline unsigned long native_read_cr2(void)
{
	unsigned long val;
	asm volatile("movl %%cr2,%0\n\t" :"=r" (val));
	return val;
}

static inline void native_write_cr2(unsigned long val)
{
	asm volatile("movl %0,%%cr2": :"r" (val));
}

static inline unsigned long native_read_cr3(void)
{
	unsigned long val;
	asm volatile("movl %%cr3,%0\n\t" :"=r" (val));
	return val;
}

static inline void native_write_cr3(unsigned long val)
{
	asm volatile("movl %0,%%cr3": :"r" (val));
}

static inline unsigned long native_read_cr4(void)
{
	unsigned long val;
	asm volatile("movl %%cr4,%0\n\t" :"=r" (val));
	return val;
}

static inline unsigned long native_read_cr4_safe(void)
{
	unsigned long val;
	/* This could fault if %cr4 does not exist */
	asm("1: movl %%cr4, %0		\n"
		"2:				\n"
		".section __ex_table,\"a\"	\n"
		".long 1b,2b			\n"
		".previous			\n"
		: "=r" (val): "0" (0));
	return val;
}

static inline void native_write_cr4(unsigned long val)
{
	asm volatile("movl %0,%%cr4": :"r" (val));
}

static inline void native_wbinvd(void)
{
	asm volatile("wbinvd": : :"memory");
}


#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#define read_cr0()	(native_read_cr0())
#define write_cr0(x)	(native_write_cr0(x))
#define read_cr2()	(native_read_cr2())
#define write_cr2(x)	(native_write_cr2(x))
#define read_cr3()	(native_read_cr3())
#define write_cr3(x)	(native_write_cr3(x))
#define read_cr4()	(native_read_cr4())
#define read_cr4_safe()	(native_read_cr4_safe())
#define write_cr4(x)	(native_write_cr4(x))
#define wbinvd()	(native_wbinvd())

/* Clear the 'TS' bit */
#define clts()		(native_clts())

#endif/* CONFIG_PARAVIRT */

/* Set the 'TS' bit */
#define stts() write_cr0(8 | read_cr0())

#endif	/* __KERNEL__ */

static inline unsigned long get_limit(unsigned long segment)
{
	unsigned long __limit;
	__asm__("lsll %1,%0"
		:"=r" (__limit):"r" (segment));
	return __limit+1;
}

#define nop() __asm__ __volatile__ ("nop")

#define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))

#define tas(ptr) (xchg((ptr),1))

struct __xchg_dummy { unsigned long a[100]; };
#define __xg(x) ((struct __xchg_dummy *)(x))


#ifdef CONFIG_X86_CMPXCHG64

/*
 * The semantics of XCHGCMP8B are a bit strange, this is why
 * there is a loop and the loading of %%eax and %%edx has to
 * be inside. This inlines well in most cases, the cached
 * cost is around ~38 cycles. (in the future we might want
 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
 * might have an implicit FPU-save as a cost, so it's not
 * clear which path to go.)
 *
 * cmpxchg8b must be used with the lock prefix here to allow
 * the instruction to be executed atomically, see page 3-102
 * of the instruction set reference 24319102.pdf. We need
 * the reader side to see the coherent 64bit value.
 */
static inline void __set_64bit (unsigned long long * ptr,
		unsigned int low, unsigned int high)
{
	__asm__ __volatile__ (
		"\n1:\t"
		"movl (%0), %%eax\n\t"
		"movl 4(%0), %%edx\n\t"
		"lock cmpxchg8b (%0)\n\t"
		"jnz 1b"
		: /* no outputs */
		:	"D"(ptr),
			"b"(low),
			"c"(high)
		:	"ax","dx","memory");
}

static inline void __set_64bit_constant (unsigned long long *ptr,
						 unsigned long long value)
{
	__set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL));
}
#define ll_low(x)	*(((unsigned int*)&(x))+0)
#define ll_high(x)	*(((unsigned int*)&(x))+1)

static inline void __set_64bit_var (unsigned long long *ptr,
			 unsigned long long value)
{
	__set_64bit(ptr,ll_low(value), ll_high(value));
}

#define set_64bit(ptr,value) \
(__builtin_constant_p(value) ? \
 __set_64bit_constant(ptr, value) : \
 __set_64bit_var(ptr, value) )

#define _set_64bit(ptr,value) \
(__builtin_constant_p(value) ? \
 __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
 __set_64bit(ptr, ll_low(value), ll_high(value)) )

#endif

/*
 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
 * Note 2: xchg has side effect, so that attribute volatile is necessary,
 *	  but generally the primitive is invalid, *ptr is output argument. --ANK
 */
static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
{
	switch (size) {
		case 1:
			__asm__ __volatile__("xchgb %b0,%1"
				:"=q" (x)
				:"m" (*__xg(ptr)), "0" (x)
				:"memory");
			break;
		case 2:
			__asm__ __volatile__("xchgw %w0,%1"
				:"=r" (x)
				:"m" (*__xg(ptr)), "0" (x)
				:"memory");
			break;
		case 4:
			__asm__ __volatile__("xchgl %0,%1"
				:"=r" (x)
				:"m" (*__xg(ptr)), "0" (x)
				:"memory");
			break;
	}
	return x;
}

/*
 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 * store NEW in MEM.  Return the initial value in MEM.  Success is
 * indicated by comparing RETURN with OLD.
 */

#ifdef CONFIG_X86_CMPXCHG
#define __HAVE_ARCH_CMPXCHG 1
#define cmpxchg(ptr,o,n)\
	((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
					(unsigned long)(n),sizeof(*(ptr))))
#define sync_cmpxchg(ptr,o,n)\
	((__typeof__(*(ptr)))__sync_cmpxchg((ptr),(unsigned long)(o),\
					(unsigned long)(n),sizeof(*(ptr))))
#endif

static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
				      unsigned long new, int size)
{
	unsigned long prev;
	switch (size) {
	case 1:
		__asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
				     : "=a"(prev)
				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
				     : "memory");
		return prev;
	case 2:
		__asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
				     : "=a"(prev)
				     : "r"(new), "m"(*__xg(ptr)), "0"(old)
				     : "memory");
		return prev;
	case 4:
		__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
				     : "=a"(prev)
				     : "r"(new), "m"(*__xg(ptr)), "0"(old)
				     : "memory");
		return prev;
	}
	return old;
}

/*
 * Always use locked operations when touching memory shared with a
 * hypervisor, since the system may be SMP even if the guest kernel
 * isn't.
 */
static inline unsigned long __sync_cmpxchg(volatile void *ptr,
					    unsigned long old,
					    unsigned long new, int size)
{
	unsigned long prev;
	switch (size) {
	case 1:
		__asm__ __volatile__("lock; cmpxchgb %b1,%2"
				     : "=a"(prev)
				     : "q"(new), "m"(*__xg(ptr)), "0"(old)
				     : "memory");
		return prev;
	case 2:
		__asm__ __volatile__("lock; cmpxchgw %w1,%2"
				     : "=a"(prev)
				     : "r"(new), "m"(*__xg(ptr)), "0"(old)
				     : "memory");
		return prev;
	case 4:
		__asm__ __volatile__("lock; cmpxchgl %1,%2"
				     : "=a"(prev)
				     : "r"(new), "m"(*__xg(ptr)), "0"(old)
				     : "memory");
		return prev;
	}
	return old;
}

#ifndef CONFIG_X86_CMPXCHG
/*
 * Building a kernel capable running on 80386. It may be necessary to
 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
 * a function for each of the sizes we support.
 */

extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);

static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
				      unsigned long new, int size)
{
	switch (size) {
	case 1:
		return cmpxchg_386_u8(ptr, old, new);
	case 2:
		return cmpxchg_386_u16(ptr, old, new);
	case 4:
		return cmpxchg_386_u32(ptr, old, new);
	}
	return old;
}

#define cmpxchg(ptr,o,n)						\
({									\
	__typeof__(*(ptr)) __ret;					\
	if (likely(boot_cpu_data.x86 > 3))				\
		__ret = __cmpxchg((ptr), (unsigned long)(o),		\
					(unsigned long)(n), sizeof(*(ptr))); \
	else								\
		__ret = cmpxchg_386((ptr), (unsigned long)(o),		\
					(unsigned long)(n), sizeof(*(ptr))); \
	__ret;								\
})
#endif

#ifdef CONFIG_X86_CMPXCHG64

static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
				      unsigned long long new)
{
	unsigned long long prev;
	__asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
			     : "=A"(prev)
			     : "b"((unsigned long)new),
			       "c"((unsigned long)(new >> 32)),
			       "m"(*__xg(ptr)),
			       "0"(old)
			     : "memory");
	return prev;
}

#define cmpxchg64(ptr,o,n)\
	((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
					(unsigned long long)(n)))

#endif
    
/*
 * Force strict CPU ordering.
 * And yes, this is required on UP too when we're talking
 * to devices.
 *
 * For now, "wmb()" doesn't actually do anything, as all
 * Intel CPU's follow what Intel calls a *Processor Order*,
 * in which all writes are seen in the program order even
 * outside the CPU.
 *
 * I expect future Intel CPU's to have a weaker ordering,
 * but I'd also expect them to finally get their act together
 * and add some real memory barriers if so.
 *
 * Some non intel clones support out of order store. wmb() ceases to be a
 * nop for these.
 */
 

/* 
 * Actually only lfence would be needed for mb() because all stores done 
 * by the kernel should be already ordered. But keep a full barrier for now. 
 */

#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)

/**
 * read_barrier_depends - Flush all pending reads that subsequents reads
 * depend on.
 *
 * No data-dependent reads from memory-like regions are ever reordered
 * over this barrier.  All reads preceding this primitive are guaranteed
 * to access memory (but not necessarily other CPUs' caches) before any
 * reads following this primitive that depend on the data return by
 * any of the preceding reads.  This primitive is much lighter weight than
 * rmb() on most CPUs, and is never heavier weight than is
 * rmb().
 *
 * These ordering constraints are respected by both the local CPU
 * and the compiler.
 *
 * Ordering is not guaranteed by anything other than these primitives,
 * not even by data dependencies.  See the documentation for
 * memory_barrier() for examples and URLs to more information.
 *
 * For example, the following code would force ordering (the initial
 * value of "a" is zero, "b" is one, and "p" is "&a"):
 *
 * <programlisting>
 *	CPU 0				CPU 1
 *
 *	b = 2;
 *	memory_barrier();
 *	p = &b;				q = p;
 *					read_barrier_depends();
 *					d = *q;
 * </programlisting>
 *
 * because the read of "*q" depends on the read of "p" and these
 * two reads are separated by a read_barrier_depends().  However,
 * the following code, with the same initial values for "a" and "b":
 *
 * <programlisting>
 *	CPU 0				CPU 1
 *
 *	a = 2;
 *	memory_barrier();
 *	b = 3;				y = b;
 *					read_barrier_depends();
 *					x = a;
 * </programlisting>
 *
 * does not enforce ordering, since there is no data dependency between
 * the read of "a" and the read of "b".  Therefore, on some CPUs, such
 * as Alpha, "y" could be set to 3 and "x" to 0.  Use rmb()
 * in cases like this where there are no data dependencies.
 **/

#define read_barrier_depends()	do { } while(0)

#ifdef CONFIG_X86_OOSTORE
/* Actually there are no OOO store capable CPUs for now that do SSE, 
   but make it already an possibility. */
#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
#else
#define wmb()	__asm__ __volatile__ ("": : :"memory")
#endif

#ifdef CONFIG_SMP
#define smp_mb()	mb()
#define smp_rmb()	rmb()
#define smp_wmb()	wmb()
#define smp_read_barrier_depends()	read_barrier_depends()
#define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
#else
#define smp_mb()	barrier()
#define smp_rmb()	barrier()
#define smp_wmb()	barrier()
#define smp_read_barrier_depends()	do { } while(0)
#define set_mb(var, value) do { var = value; barrier(); } while (0)
#endif

#include <linux/irqflags.h>

/*
 * disable hlt during certain critical i/o operations
 */
#define HAVE_DISABLE_HLT
void disable_hlt(void);
void enable_hlt(void);

extern int es7000_plat;
void cpu_idle_wait(void);

/*
 * On SMP systems, when the scheduler does migration-cost autodetection,
 * it needs a way to flush as much of the CPU's caches as possible:
 */
static inline void sched_cacheflush(void)
{
	wbinvd();
}

extern unsigned long arch_align_stack(unsigned long sp);
extern void free_init_pages(char *what, unsigned long begin, unsigned long end);

void default_idle(void);

#endif
Commit	Line	Data
1da177e4 LT	1	#ifndef __ASM_SYSTEM_H
	2	#define __ASM_SYSTEM_H
	3
1da177e4 LT	4	#include <linux/kernel.h>
	5	#include <asm/segment.h>
	6	#include <asm/cpufeature.h>
	7	#include <linux/bitops.h> /* for LOCK_PREFIX */
	8
	9	#ifdef __KERNEL__
	10
	11	struct task_struct; /* one of the stranger aspects of C forward declarations.. */
	12	extern struct task_struct * FASTCALL(__switch_to(struct task_struct prev, struct task_struct next));
	13
47a5c6fa LT	14	/*
	15	* Saving eflags is important. It switches not only IOPL between tasks,
	16	* it also protects other tasks from NT leaking through sysenter etc.
	17	*/
1da177e4 LT	18	#define switch_to(prev,next,last) do { \
1da177e4 LT	19	unsigned long esi,edi; \
47a5c6fa LT	20	asm volatile("pushfl\n\t" /* Save flags */ \
47a5c6fa LT	21	"pushl %%ebp\n\t" \
1da177e4 LT	22	"movl %%esp,%0\n\t" /* save ESP */ \
	23	"movl %5,%%esp\n\t" /* restore ESP */ \
	24	"movl $1f,%1\n\t" /* save EIP */ \
	25	"pushl %6\n\t" /* restore EIP */ \
	26	"jmp __switch_to\n" \
	27	"1:\t" \
	28	"popl %%ebp\n\t" \
47a5c6fa	29	"popfl" \
1da177e4 LT	30	:"=m" (prev->thread.esp),"=m" (prev->thread.eip), \
	31	"=a" (last),"=S" (esi),"=D" (edi) \
	32	:"m" (next->thread.esp),"m" (next->thread.eip), \
	33	"2" (prev), "d" (next)); \
	34	} while (0)
	35
	36	#define _set_base(addr,base) do { unsigned long __pr; \
	37	__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	38	"rorl $16,%%edx\n\t" \
	39	"movb %%dl,%2\n\t" \
	40	"movb %%dh,%3" \
	41	:"=&d" (__pr) \
	42	:"m" (*((addr)+2)), \
	43	"m" (*((addr)+4)), \
	44	"m" (*((addr)+7)), \
	45	"0" (base) \
	46	); } while(0)
	47
	48	#define _set_limit(addr,limit) do { unsigned long __lr; \
	49	__asm__ __volatile__ ("movw %%dx,%1\n\t" \
	50	"rorl $16,%%edx\n\t" \
	51	"movb %2,%%dh\n\t" \
	52	"andb $0xf0,%%dh\n\t" \
	53	"orb %%dh,%%dl\n\t" \
	54	"movb %%dl,%2" \
	55	:"=&d" (__lr) \
	56	:"m" (*(addr)), \
	57	"m" (*((addr)+6)), \
	58	"0" (limit) \
	59	); } while(0)
	60
	61	#define set_base(ldt,base) _set_base( ((char *)&(ldt)) , (base) )
5fe9fe3c	62	#define set_limit(ldt,limit) _set_limit( ((char *)&(ldt)) , ((limit)-1) )
1da177e4	63
1da177e4 LT	64	/*
	65	* Load a segment. Fall back on loading the zero
	66	* segment if something goes wrong..
	67	*/
	68	#define loadsegment(seg,value) \
	69	asm volatile("\n" \
	70	"1:\t" \
fd51f666	71	"mov %0,%%" #seg "\n" \
1da177e4 LT	72	"2:\n" \
	73	".section .fixup,\"ax\"\n" \
	74	"3:\t" \
	75	"pushl $0\n\t" \
	76	"popl %%" #seg "\n\t" \
	77	"jmp 2b\n" \
	78	".previous\n" \
	79	".section __ex_table,\"a\"\n\t" \
	80	".align 4\n\t" \
	81	".long 1b,3b\n" \
	82	".previous" \
4d37e7e3	83	: :"rm" (value))
1da177e4 LT	84
	85	/*
	86	* Save a segment register away
	87	*/
	88	#define savesegment(seg, value) \
4d37e7e3	89	asm volatile("mov %%" #seg ",%0":"=rm" (value))
1da177e4	90
90a0a06a RR	91
	92	static inline void native_clts(void)
	93	{
	94	asm volatile ("clts");
	95	}
	96
	97	static inline unsigned long native_read_cr0(void)
	98	{
	99	unsigned long val;
	100	asm volatile("movl %%cr0,%0\n\t" :"=r" (val));
	101	return val;
	102	}
	103
	104	static inline void native_write_cr0(unsigned long val)
	105	{
	106	asm volatile("movl %0,%%cr0": :"r" (val));
	107	}
	108
	109	static inline unsigned long native_read_cr2(void)
	110	{
	111	unsigned long val;
	112	asm volatile("movl %%cr2,%0\n\t" :"=r" (val));
	113	return val;
	114	}
	115
	116	static inline void native_write_cr2(unsigned long val)
	117	{
	118	asm volatile("movl %0,%%cr2": :"r" (val));
	119	}
	120
	121	static inline unsigned long native_read_cr3(void)
	122	{
	123	unsigned long val;
	124	asm volatile("movl %%cr3,%0\n\t" :"=r" (val));
	125	return val;
	126	}
	127
	128	static inline void native_write_cr3(unsigned long val)
	129	{
	130	asm volatile("movl %0,%%cr3": :"r" (val));
	131	}
	132
	133	static inline unsigned long native_read_cr4(void)
	134	{
	135	unsigned long val;
	136	asm volatile("movl %%cr4,%0\n\t" :"=r" (val));
	137	return val;
	138	}
	139
	140	static inline unsigned long native_read_cr4_safe(void)
	141	{
	142	unsigned long val;
	143	/* This could fault if %cr4 does not exist */
	144	asm("1: movl %%cr4, %0 \n"
	145	"2: \n"
	146	".section __ex_table,\"a\" \n"
	147	".long 1b,2b \n"
	148	".previous \n"
	149	: "=r" (val): "0" (0));
	150	return val;
	151	}
	152
	153	static inline void native_write_cr4(unsigned long val)
	154	{
155	asm volatile("movl %0,%%cr4": :"r" (val));
156	}
157
158	static inline void native_wbinvd(void)
159	{
160	asm volatile("wbinvd": : :"memory");
161	}
162
163
d3561b7f RR	164	#ifdef CONFIG_PARAVIRT
	165	#include <asm/paravirt.h>
	166	#else
90a0a06a RR	167	#define read_cr0() (native_read_cr0())
	168	#define write_cr0(x) (native_write_cr0(x))
	169	#define read_cr2() (native_read_cr2())
	170	#define write_cr2(x) (native_write_cr2(x))
	171	#define read_cr3() (native_read_cr3())
	172	#define write_cr3(x) (native_write_cr3(x))
	173	#define read_cr4() (native_read_cr4())
	174	#define read_cr4_safe() (native_read_cr4_safe())
	175	#define write_cr4(x) (native_write_cr4(x))
	176	#define wbinvd() (native_wbinvd())
d3561b7f RR	177
d3561b7f RR	178	/* Clear the 'TS' bit */
90a0a06a RR	179	#define clts() (native_clts())
90a0a06a RR	180
d3561b7f RR	181	#endif/* CONFIG_PARAVIRT */
	182
	183	/* Set the 'TS' bit */
1da177e4 LT	184	#define stts() write_cr0(8 \| read_cr0())
	185
	186	#endif /* __KERNEL__ */
	187
1da177e4 LT	188	static inline unsigned long get_limit(unsigned long segment)
	189	{
	190	unsigned long __limit;
	191	__asm__("lsll %1,%0"
	192	:"=r" (__limit):"r" (segment));
	193	return __limit+1;
	194	}
	195
	196	#define nop() __asm__ __volatile__ ("nop")
	197
	198	#define xchg(ptr,v) ((__typeof__((ptr)))__xchg((unsigned long)(v),(ptr),sizeof((ptr))))
	199
	200	#define tas(ptr) (xchg((ptr),1))
	201
	202	struct __xchg_dummy { unsigned long a[100]; };
	203	#define __xg(x) ((struct __xchg_dummy *)(x))
	204
	205
8896fab3 JB	206	#ifdef CONFIG_X86_CMPXCHG64
8896fab3 JB	207
1da177e4 LT	208	/*
	209	* The semantics of XCHGCMP8B are a bit strange, this is why
	210	* there is a loop and the loading of %%eax and %%edx has to
	211	* be inside. This inlines well in most cases, the cached
	212	* cost is around ~38 cycles. (in the future we might want
	213	* to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
	214	* might have an implicit FPU-save as a cost, so it's not
	215	* clear which path to go.)
	216	*
	217	* cmpxchg8b must be used with the lock prefix here to allow
	218	* the instruction to be executed atomically, see page 3-102
	219	* of the instruction set reference 24319102.pdf. We need
	220	* the reader side to see the coherent 64bit value.
	221	*/
	222	static inline void __set_64bit (unsigned long long * ptr,
	223	unsigned int low, unsigned int high)
	224	{
	225	__asm__ __volatile__ (
	226	"\n1:\t"
	227	"movl (%0), %%eax\n\t"
	228	"movl 4(%0), %%edx\n\t"
	229	"lock cmpxchg8b (%0)\n\t"
	230	"jnz 1b"
	231	: /* no outputs */
	232	: "D"(ptr),
	233	"b"(low),
	234	"c"(high)
	235	: "ax","dx","memory");
	236	}
	237
	238	static inline void __set_64bit_constant (unsigned long long *ptr,
	239	unsigned long long value)
	240	{
	241	__set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL));
	242	}
	243	#define ll_low(x) (((unsigned int)&(x))+0)
	244	#define ll_high(x) (((unsigned int)&(x))+1)
	245
	246	static inline void __set_64bit_var (unsigned long long *ptr,
	247	unsigned long long value)
	248	{
	249	__set_64bit(ptr,ll_low(value), ll_high(value));
	250	}
	251
	252	#define set_64bit(ptr,value) \
	253	(__builtin_constant_p(value) ? \
	254	__set_64bit_constant(ptr, value) : \
	255	__set_64bit_var(ptr, value) )
	256
	257	#define _set_64bit(ptr,value) \
	258	(__builtin_constant_p(value) ? \
	259	__set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
	260	__set_64bit(ptr, ll_low(value), ll_high(value)) )
	261
8896fab3 JB	262	#endif
8896fab3 JB	263
1da177e4 LT	264	/*
	265	* Note: no "lock" prefix even on SMP: xchg always implies lock anyway
	266	* Note 2: xchg has side effect, so that attribute volatile is necessary,
	267	* but generally the primitive is invalid, *ptr is output argument. --ANK
	268	*/
	269	static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
	270	{
	271	switch (size) {
	272	case 1:
	273	__asm__ __volatile__("xchgb %b0,%1"
	274	:"=q" (x)
	275	:"m" (*__xg(ptr)), "0" (x)
	276	:"memory");
	277	break;
	278	case 2:
	279	__asm__ __volatile__("xchgw %w0,%1"
	280	:"=r" (x)
	281	:"m" (*__xg(ptr)), "0" (x)
	282	:"memory");
	283	break;
	284	case 4:
	285	__asm__ __volatile__("xchgl %0,%1"
	286	:"=r" (x)
	287	:"m" (*__xg(ptr)), "0" (x)
	288	:"memory");
	289	break;
	290	}
	291	return x;
	292	}
	293
	294	/*
	295	* Atomic compare and exchange. Compare OLD with MEM, if identical,
	296	* store NEW in MEM. Return the initial value in MEM. Success is
	297	* indicated by comparing RETURN with OLD.
	298	*/
	299
	300	#ifdef CONFIG_X86_CMPXCHG
	301	#define __HAVE_ARCH_CMPXCHG 1
53e86b91 NP	302	#define cmpxchg(ptr,o,n)\
	303	((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
	304	(unsigned long)(n),sizeof(*(ptr))))
027a8c7e CW	305	#define sync_cmpxchg(ptr,o,n)\
	306	((__typeof__(*(ptr)))__sync_cmpxchg((ptr),(unsigned long)(o),\
	307	(unsigned long)(n),sizeof(*(ptr))))
53e86b91	308	#endif
1da177e4 LT	309
	310	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
	311	unsigned long new, int size)
	312	{
	313	unsigned long prev;
	314	switch (size) {
	315	case 1:
	316	__asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
	317	: "=a"(prev)
	318	: "q"(new), "m"(*__xg(ptr)), "0"(old)
	319	: "memory");
	320	return prev;
	321	case 2:
	322	__asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
	323	: "=a"(prev)
8896fab3	324	: "r"(new), "m"(*__xg(ptr)), "0"(old)
1da177e4 LT	325	: "memory");
	326	return prev;
	327	case 4:
	328	__asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2"
	329	: "=a"(prev)
8896fab3	330	: "r"(new), "m"(*__xg(ptr)), "0"(old)
1da177e4 LT	331	: "memory");
	332	return prev;
	333	}
	334	return old;
	335	}
	336
027a8c7e CW	337	/*
	338	* Always use locked operations when touching memory shared with a
	339	* hypervisor, since the system may be SMP even if the guest kernel
	340	* isn't.
	341	*/
	342	static inline unsigned long __sync_cmpxchg(volatile void *ptr,
	343	unsigned long old,
	344	unsigned long new, int size)
	345	{
	346	unsigned long prev;
	347	switch (size) {
	348	case 1:
	349	__asm__ __volatile__("lock; cmpxchgb %b1,%2"
	350	: "=a"(prev)
	351	: "q"(new), "m"(*__xg(ptr)), "0"(old)
	352	: "memory");
	353	return prev;
	354	case 2:
	355	__asm__ __volatile__("lock; cmpxchgw %w1,%2"
	356	: "=a"(prev)
	357	: "r"(new), "m"(*__xg(ptr)), "0"(old)
	358	: "memory");
	359	return prev;
	360	case 4:
	361	__asm__ __volatile__("lock; cmpxchgl %1,%2"
	362	: "=a"(prev)
	363	: "r"(new), "m"(*__xg(ptr)), "0"(old)
	364	: "memory");
	365	return prev;
	366	}
	367	return old;
	368	}
	369
53e86b91 NP	370	#ifndef CONFIG_X86_CMPXCHG
	371	/*
	372	* Building a kernel capable running on 80386. It may be necessary to
	373	* simulate the cmpxchg on the 80386 CPU. For that purpose we define
	374	* a function for each of the sizes we support.
	375	*/
8896fab3	376
53e86b91 NP	377	extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
	378	extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
	379	extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
	380
	381	static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
	382	unsigned long new, int size)
	383	{
	384	switch (size) {
	385	case 1:
	386	return cmpxchg_386_u8(ptr, old, new);
	387	case 2:
	388	return cmpxchg_386_u16(ptr, old, new);
	389	case 4:
	390	return cmpxchg_386_u32(ptr, old, new);
	391	}
	392	return old;
	393	}
	394
	395	#define cmpxchg(ptr,o,n) \
	396	({ \
	397	__typeof__(*(ptr)) __ret; \
	398	if (likely(boot_cpu_data.x86 > 3)) \
	399	__ret = __cmpxchg((ptr), (unsigned long)(o), \
	400	(unsigned long)(n), sizeof(*(ptr))); \
	401	else \
	402	__ret = cmpxchg_386((ptr), (unsigned long)(o), \
	403	(unsigned long)(n), sizeof(*(ptr))); \
	404	__ret; \
	405	})
8896fab3 JB	406	#endif
	407
	408	#ifdef CONFIG_X86_CMPXCHG64
	409
	410	static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long long old,
	411	unsigned long long new)
	412	{
	413	unsigned long long prev;
	414	__asm__ __volatile__(LOCK_PREFIX "cmpxchg8b %3"
	415	: "=A"(prev)
	416	: "b"((unsigned long)new),
	417	"c"((unsigned long)(new >> 32)),
	418	"m"(*__xg(ptr)),
	419	"0"(old)
	420	: "memory");
	421	return prev;
	422	}
	423
	424	#define cmpxchg64(ptr,o,n)\
	425	((__typeof__(*(ptr)))__cmpxchg64((ptr),(unsigned long long)(o),\
	426	(unsigned long long)(n)))
	427
	428	#endif
1da177e4	429
1da177e4 LT	430	/*
	431	* Force strict CPU ordering.
	432	* And yes, this is required on UP too when we're talking
	433	* to devices.
	434	*
	435	* For now, "wmb()" doesn't actually do anything, as all
	436	* Intel CPU's follow what Intel calls a Processor Order,
	437	* in which all writes are seen in the program order even
	438	* outside the CPU.
	439	*
	440	* I expect future Intel CPU's to have a weaker ordering,
	441	* but I'd also expect them to finally get their act together
	442	* and add some real memory barriers if so.
	443	*
	444	* Some non intel clones support out of order store. wmb() ceases to be a
	445	* nop for these.
	446	*/
	447
	448
	449	/*
	450	* Actually only lfence would be needed for mb() because all stores done
	451	* by the kernel should be already ordered. But keep a full barrier for now.
	452	*/
	453
	454	#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
	455	#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
	456
	457	/**
	458	* read_barrier_depends - Flush all pending reads that subsequents reads
	459	* depend on.
	460	*
	461	* No data-dependent reads from memory-like regions are ever reordered
	462	* over this barrier. All reads preceding this primitive are guaranteed
	463	* to access memory (but not necessarily other CPUs' caches) before any
	464	* reads following this primitive that depend on the data return by
	465	* any of the preceding reads. This primitive is much lighter weight than
	466	* rmb() on most CPUs, and is never heavier weight than is
	467	* rmb().
	468	*
	469	* These ordering constraints are respected by both the local CPU
	470	* and the compiler.
	471	*
	472	* Ordering is not guaranteed by anything other than these primitives,
	473	* not even by data dependencies. See the documentation for
	474	* memory_barrier() for examples and URLs to more information.
	475	*
	476	* For example, the following code would force ordering (the initial
	477	* value of "a" is zero, "b" is one, and "p" is "&a"):
	478	*
	479	* <programlisting>
	480	* CPU 0 CPU 1
	481	*
	482	* b = 2;
	483	* memory_barrier();
	484	* p = &b; q = p;
	485	* read_barrier_depends();
	486	* d = *q;
	487	* </programlisting>
	488	*
	489	* because the read of "*q" depends on the read of "p" and these
	490	* two reads are separated by a read_barrier_depends(). However,
	491	* the following code, with the same initial values for "a" and "b":
	492	*
	493	* <programlisting>
494	* CPU 0 CPU 1
495	*
496	* a = 2;
497	* memory_barrier();
498	* b = 3; y = b;
499	* read_barrier_depends();
500	* x = a;
501	* </programlisting>
502	*
503	* does not enforce ordering, since there is no data dependency between
504	* the read of "a" and the read of "b". Therefore, on some CPUs, such
505	* as Alpha, "y" could be set to 3 and "x" to 0. Use rmb()
d6e05edc	506	* in cases like this where there are no data dependencies.
1da177e4 LT	507	**/
	508
	509	#define read_barrier_depends() do { } while(0)
	510
	511	#ifdef CONFIG_X86_OOSTORE
	512	/* Actually there are no OOO store capable CPUs for now that do SSE,
	513	but make it already an possibility. */
	514	#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
	515	#else
	516	#define wmb() __asm__ __volatile__ ("": : :"memory")
	517	#endif
	518
	519	#ifdef CONFIG_SMP
	520	#define smp_mb() mb()
	521	#define smp_rmb() rmb()
	522	#define smp_wmb() wmb()
	523	#define smp_read_barrier_depends() read_barrier_depends()
911b0ad2	524	#define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
1da177e4 LT	525	#else
	526	#define smp_mb() barrier()
	527	#define smp_rmb() barrier()
	528	#define smp_wmb() barrier()
	529	#define smp_read_barrier_depends() do { } while(0)
	530	#define set_mb(var, value) do { var = value; barrier(); } while (0)
	531	#endif
	532
55f327fa	533	#include <linux/irqflags.h>
1da177e4 LT	534
	535	/*
	536	* disable hlt during certain critical i/o operations
	537	*/
	538	#define HAVE_DISABLE_HLT
	539	void disable_hlt(void);
	540	void enable_hlt(void);
	541
	542	extern int es7000_plat;
	543	void cpu_idle_wait(void);
	544
4dc7a0bb IM	545	/*
	546	* On SMP systems, when the scheduler does migration-cost autodetection,
	547	* it needs a way to flush as much of the CPU's caches as possible:
	548	*/
	549	static inline void sched_cacheflush(void)
	550	{
	551	wbinvd();
	552	}
	553
1da177e4	554	extern unsigned long arch_align_stack(unsigned long sp);
9a0b5817	555	extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
1da177e4	556
cdb04527 AB	557	void default_idle(void);
cdb04527 AB	558
1da177e4	559	#endif