[GitHub/mt8127/android_kernel_alcatel_ttab.git] / kernel / spinlock.c

/*
 * Copyright (2004) Linus Torvalds
 *
 * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
 *
 * Copyright (2004, 2005) Ingo Molnar
 *
 * This file contains the spinlock/rwlock implementations for the
 * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
 *
 * Note that some architectures have special knowledge about the
 * stack frames of these functions in their profile_pc. If you
 * change anything significant here that could change the stack
 * frame contact the architecture maintainers.
 */

#include <linux/linkage.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/debug_locks.h>
#include <linux/module.h>

/*
 * Generic declaration of the raw read_trylock() function,
 * architectures are supposed to optimize this:
 */
int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock)
{
	__raw_read_lock(lock);
	return 1;
}
EXPORT_SYMBOL(generic__raw_read_trylock);

int __lockfunc _spin_trylock(spinlock_t *lock)
{
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}
	
	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_spin_trylock);

int __lockfunc _read_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_read_trylock(lock)) {
		rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_read_trylock);

int __lockfunc _write_trylock(rwlock_t *lock)
{
	preempt_disable();
	if (_raw_write_trylock(lock)) {
		rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(_write_trylock);

/*
 * If lockdep is enabled then we use the non-preemption spin-ops
 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
 */
#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
	defined(CONFIG_DEBUG_LOCK_ALLOC)

void __lockfunc _read_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock);

unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	/*
	 * On lockdep we dont want the hand-coded irq-enable of
	 * _raw_spin_lock_flags() code, because lockdep assumes
	 * that interrupts are not re-enabled during lock-acquire:
	 */
#ifdef CONFIG_PROVE_LOCKING
	_raw_spin_lock(lock);
#else
	_raw_spin_lock_flags(lock, &flags);
#endif
	return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave);

void __lockfunc _spin_lock_irq(spinlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_spin_lock(lock);
}
EXPORT_SYMBOL(_spin_lock_irq);

void __lockfunc _spin_lock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_spin_lock(lock);
}
EXPORT_SYMBOL(_spin_lock_bh);

unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
	return flags;
}
EXPORT_SYMBOL(_read_lock_irqsave);

void __lockfunc _read_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock_irq);

void __lockfunc _read_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock_bh);

unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
{
	unsigned long flags;

	local_irq_save(flags);
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
	return flags;
}
EXPORT_SYMBOL(_write_lock_irqsave);

void __lockfunc _write_lock_irq(rwlock_t *lock)
{
	local_irq_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
}
EXPORT_SYMBOL(_write_lock_irq);

void __lockfunc _write_lock_bh(rwlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
}
EXPORT_SYMBOL(_write_lock_bh);

void __lockfunc _spin_lock(spinlock_t *lock)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_spin_lock(lock);
}

EXPORT_SYMBOL(_spin_lock);

void __lockfunc _write_lock(rwlock_t *lock)
{
	preempt_disable();
	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	_raw_write_lock(lock);
}

EXPORT_SYMBOL(_write_lock);

#else /* CONFIG_PREEMPT: */

/*
 * This could be a long-held lock. We both prepare to spin for a long
 * time (making _this_ CPU preemptable if possible), and we also signal
 * towards that other CPU that it should break the lock ASAP.
 *
 * (We do this in a function because inlining it would be excessive.)
 */

#define BUILD_LOCK_OPS(op, locktype)					\
void __lockfunc _##op##_lock(locktype##_t *lock)			\
{									\
	for (;;) {							\
		preempt_disable();					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			cpu_relax();					\
	}								\
	(lock)->break_lock = 0;						\
}									\
									\
EXPORT_SYMBOL(_##op##_lock);						\
									\
unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)	\
{									\
	unsigned long flags;						\
									\
	for (;;) {							\
		preempt_disable();					\
		local_irq_save(flags);					\
		if (likely(_raw_##op##_trylock(lock)))			\
			break;						\
		local_irq_restore(flags);				\
		preempt_enable();					\
									\
		if (!(lock)->break_lock)				\
			(lock)->break_lock = 1;				\
		while (!op##_can_lock(lock) && (lock)->break_lock)	\
			cpu_relax();					\
	}								\
	(lock)->break_lock = 0;						\
	return flags;							\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irqsave);					\
									\
void __lockfunc _##op##_lock_irq(locktype##_t *lock)			\
{									\
	_##op##_lock_irqsave(lock);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_irq);					\
									\
void __lockfunc _##op##_lock_bh(locktype##_t *lock)			\
{									\
	unsigned long flags;						\
									\
	/*							*/	\
	/* Careful: we must exclude softirqs too, hence the	*/	\
	/* irq-disabling. We use the generic preemption-aware	*/	\
	/* function:						*/	\
	/**/								\
	flags = _##op##_lock_irqsave(lock);				\
	local_bh_disable();						\
	local_irq_restore(flags);					\
}									\
									\
EXPORT_SYMBOL(_##op##_lock_bh)

/*
 * Build preemption-friendly versions of the following
 * lock-spinning functions:
 *
 *         _[spin|read|write]_lock()
 *         _[spin|read|write]_lock_irq()
 *         _[spin|read|write]_lock_irqsave()
 *         _[spin|read|write]_lock_bh()
 */
BUILD_LOCK_OPS(spin, spinlock);
BUILD_LOCK_OPS(read, rwlock);
BUILD_LOCK_OPS(write, rwlock);

#endif /* CONFIG_PREEMPT */

#ifdef CONFIG_DEBUG_LOCK_ALLOC

void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
{
	preempt_disable();
	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	_raw_spin_lock(lock);
}

EXPORT_SYMBOL(_spin_lock_nested);

#endif

void __lockfunc _spin_unlock(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock);

void __lockfunc _write_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock);

void __lockfunc _read_unlock(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock);

void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irqrestore);

void __lockfunc _spin_unlock_irq(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irq);

void __lockfunc _spin_unlock_bh(spinlock_t *lock)
{
	spin_release(&lock->dep_map, 1, _RET_IP_);
	_raw_spin_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_spin_unlock_bh);

void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irqrestore);

void __lockfunc _read_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irq);

void __lockfunc _read_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_read_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_read_unlock_bh);

void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_restore(flags);
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irqrestore);

void __lockfunc _write_unlock_irq(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	local_irq_enable();
	preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irq);

void __lockfunc _write_unlock_bh(rwlock_t *lock)
{
	rwlock_release(&lock->dep_map, 1, _RET_IP_);
	_raw_write_unlock(lock);
	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(_write_unlock_bh);

int __lockfunc _spin_trylock_bh(spinlock_t *lock)
{
	local_bh_disable();
	preempt_disable();
	if (_raw_spin_trylock(lock)) {
		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
		return 1;
	}

	preempt_enable_no_resched();
	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	return 0;
}
EXPORT_SYMBOL(_spin_trylock_bh);

int in_lock_functions(unsigned long addr)
{
	/* Linker adds these: start and end of __lockfunc functions */
	extern char __lock_text_start[], __lock_text_end[];

	return addr >= (unsigned long)__lock_text_start
	&& addr < (unsigned long)__lock_text_end;
}
EXPORT_SYMBOL(in_lock_functions);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Copyright (2004) Linus Torvalds
	3	*
	4	* Author: Zwane Mwaikambo <zwane@fsmlabs.com>
	5	*
fb1c8f93 IM	6	* Copyright (2004, 2005) Ingo Molnar
	7	*
	8	* This file contains the spinlock/rwlock implementations for the
	9	* SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
0cb91a22 AK	10	*
	11	* Note that some architectures have special knowledge about the
	12	* stack frames of these functions in their profile_pc. If you
	13	* change anything significant here that could change the stack
	14	* frame contact the architecture maintainers.
1da177e4 LT	15	*/
1da177e4 LT	16
1da177e4 LT	17	#include <linux/linkage.h>
	18	#include <linux/preempt.h>
	19	#include <linux/spinlock.h>
	20	#include <linux/interrupt.h>
8a25d5de	21	#include <linux/debug_locks.h>
1da177e4 LT	22	#include <linux/module.h>
	23
	24	/*
	25	* Generic declaration of the raw read_trylock() function,
	26	* architectures are supposed to optimize this:
	27	*/
fb1c8f93	28	int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock)
1da177e4	29	{
fb1c8f93	30	__raw_read_lock(lock);
1da177e4 LT	31	return 1;
1da177e4 LT	32	}
fb1c8f93	33	EXPORT_SYMBOL(generic__raw_read_trylock);
1da177e4 LT	34
	35	int __lockfunc _spin_trylock(spinlock_t *lock)
	36	{
	37	preempt_disable();
8a25d5de IM	38	if (_raw_spin_trylock(lock)) {
8a25d5de IM	39	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	40	return 1;
8a25d5de	41	}
1da177e4 LT	42
	43	preempt_enable();
	44	return 0;
	45	}
	46	EXPORT_SYMBOL(_spin_trylock);
	47
	48	int __lockfunc _read_trylock(rwlock_t *lock)
	49	{
	50	preempt_disable();
8a25d5de IM	51	if (_raw_read_trylock(lock)) {
8a25d5de IM	52	rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	53	return 1;
8a25d5de	54	}
1da177e4 LT	55
	56	preempt_enable();
	57	return 0;
	58	}
	59	EXPORT_SYMBOL(_read_trylock);
	60
	61	int __lockfunc _write_trylock(rwlock_t *lock)
	62	{
	63	preempt_disable();
8a25d5de IM	64	if (_raw_write_trylock(lock)) {
8a25d5de IM	65	rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	66	return 1;
8a25d5de	67	}
1da177e4 LT	68
	69	preempt_enable();
	70	return 0;
	71	}
	72	EXPORT_SYMBOL(_write_trylock);
	73
8a25d5de IM	74	/*
	75	* If lockdep is enabled then we use the non-preemption spin-ops
	76	* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
	77	* not re-enabled during lock-acquire (which the preempt-spin-ops do):
	78	*/
	79	#if !defined(CONFIG_PREEMPT) \|\| !defined(CONFIG_SMP) \|\| \
fc47e7b5	80	defined(CONFIG_DEBUG_LOCK_ALLOC)
1da177e4 LT	81
	82	void __lockfunc _read_lock(rwlock_t *lock)
	83	{
	84	preempt_disable();
8a25d5de	85	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	86	_raw_read_lock(lock);
	87	}
	88	EXPORT_SYMBOL(_read_lock);
	89
	90	unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
	91	{
	92	unsigned long flags;
	93
	94	local_irq_save(flags);
	95	preempt_disable();
8a25d5de IM	96	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
	97	/*
	98	* On lockdep we dont want the hand-coded irq-enable of
	99	* _raw_spin_lock_flags() code, because lockdep assumes
	100	* that interrupts are not re-enabled during lock-acquire:
	101	*/
	102	#ifdef CONFIG_PROVE_LOCKING
	103	_raw_spin_lock(lock);
	104	#else
fb1c8f93	105	_raw_spin_lock_flags(lock, &flags);
8a25d5de	106	#endif
1da177e4 LT	107	return flags;
	108	}
	109	EXPORT_SYMBOL(_spin_lock_irqsave);
	110
	111	void __lockfunc _spin_lock_irq(spinlock_t *lock)
	112	{
	113	local_irq_disable();
	114	preempt_disable();
8a25d5de	115	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	116	_raw_spin_lock(lock);
	117	}
	118	EXPORT_SYMBOL(_spin_lock_irq);
	119
	120	void __lockfunc _spin_lock_bh(spinlock_t *lock)
	121	{
	122	local_bh_disable();
	123	preempt_disable();
8a25d5de	124	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	125	_raw_spin_lock(lock);
	126	}
	127	EXPORT_SYMBOL(_spin_lock_bh);
	128
	129	unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
	130	{
	131	unsigned long flags;
	132
	133	local_irq_save(flags);
	134	preempt_disable();
8a25d5de	135	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	136	_raw_read_lock(lock);
	137	return flags;
	138	}
	139	EXPORT_SYMBOL(_read_lock_irqsave);
	140
	141	void __lockfunc _read_lock_irq(rwlock_t *lock)
	142	{
	143	local_irq_disable();
	144	preempt_disable();
8a25d5de	145	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	146	_raw_read_lock(lock);
	147	}
	148	EXPORT_SYMBOL(_read_lock_irq);
	149
	150	void __lockfunc _read_lock_bh(rwlock_t *lock)
	151	{
	152	local_bh_disable();
	153	preempt_disable();
8a25d5de	154	rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	155	_raw_read_lock(lock);
	156	}
	157	EXPORT_SYMBOL(_read_lock_bh);
	158
	159	unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
	160	{
	161	unsigned long flags;
	162
	163	local_irq_save(flags);
	164	preempt_disable();
8a25d5de	165	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	166	_raw_write_lock(lock);
	167	return flags;
	168	}
	169	EXPORT_SYMBOL(_write_lock_irqsave);
	170
	171	void __lockfunc _write_lock_irq(rwlock_t *lock)
	172	{
	173	local_irq_disable();
	174	preempt_disable();
8a25d5de	175	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	176	_raw_write_lock(lock);
	177	}
	178	EXPORT_SYMBOL(_write_lock_irq);
	179
	180	void __lockfunc _write_lock_bh(rwlock_t *lock)
	181	{
	182	local_bh_disable();
	183	preempt_disable();
8a25d5de	184	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	185	_raw_write_lock(lock);
	186	}
	187	EXPORT_SYMBOL(_write_lock_bh);
	188
	189	void __lockfunc _spin_lock(spinlock_t *lock)
	190	{
	191	preempt_disable();
8a25d5de	192	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	193	_raw_spin_lock(lock);
	194	}
	195
	196	EXPORT_SYMBOL(_spin_lock);
	197
	198	void __lockfunc _write_lock(rwlock_t *lock)
	199	{
	200	preempt_disable();
8a25d5de	201	rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1da177e4 LT	202	_raw_write_lock(lock);
	203	}
	204
	205	EXPORT_SYMBOL(_write_lock);
	206
	207	#else /* CONFIG_PREEMPT: */
	208
	209	/*
	210	* This could be a long-held lock. We both prepare to spin for a long
	211	* time (making _this_ CPU preemptable if possible), and we also signal
	212	* towards that other CPU that it should break the lock ASAP.
	213	*
	214	* (We do this in a function because inlining it would be excessive.)
	215	*/
	216
	217	#define BUILD_LOCK_OPS(op, locktype) \
	218	void __lockfunc _##op##_lock(locktype##_t *lock) \
	219	{ \
1da177e4	220	for (;;) { \
ee25e96f	221	preempt_disable(); \
1da177e4 LT	222	if (likely(_raw_##op##_trylock(lock))) \
	223	break; \
	224	preempt_enable(); \
ee25e96f	225	\
1da177e4 LT	226	if (!(lock)->break_lock) \
	227	(lock)->break_lock = 1; \
	228	while (!op##_can_lock(lock) && (lock)->break_lock) \
	229	cpu_relax(); \
1da177e4 LT	230	} \
	231	(lock)->break_lock = 0; \
	232	} \
	233	\
	234	EXPORT_SYMBOL(_##op##_lock); \
	235	\
	236	unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \
	237	{ \
	238	unsigned long flags; \
	239	\
1da177e4	240	for (;;) { \
ee25e96f	241	preempt_disable(); \
1da177e4 LT	242	local_irq_save(flags); \
	243	if (likely(_raw_##op##_trylock(lock))) \
	244	break; \
	245	local_irq_restore(flags); \
1da177e4	246	preempt_enable(); \
ee25e96f	247	\
1da177e4 LT	248	if (!(lock)->break_lock) \
	249	(lock)->break_lock = 1; \
	250	while (!op##_can_lock(lock) && (lock)->break_lock) \
	251	cpu_relax(); \
1da177e4 LT	252	} \
	253	(lock)->break_lock = 0; \
	254	return flags; \
	255	} \
	256	\
	257	EXPORT_SYMBOL(_##op##_lock_irqsave); \
	258	\
	259	void __lockfunc _##op##_lock_irq(locktype##_t *lock) \
	260	{ \
	261	_##op##_lock_irqsave(lock); \
	262	} \
	263	\
	264	EXPORT_SYMBOL(_##op##_lock_irq); \
	265	\
	266	void __lockfunc _##op##_lock_bh(locktype##_t *lock) \
	267	{ \
	268	unsigned long flags; \
	269	\
	270	/* */ \
	271	/* Careful: we must exclude softirqs too, hence the */ \
	272	/* irq-disabling. We use the generic preemption-aware */ \
	273	/* function: */ \
	274	/**/ \
	275	flags = _##op##_lock_irqsave(lock); \
	276	local_bh_disable(); \
	277	local_irq_restore(flags); \
	278	} \
	279	\
	280	EXPORT_SYMBOL(_##op##_lock_bh)
	281
	282	/*
	283	* Build preemption-friendly versions of the following
	284	* lock-spinning functions:
	285	*
	286	* _[spin\|read\|write]_lock()
	287	* _[spin\|read\|write]_lock_irq()
	288	* _[spin\|read\|write]_lock_irqsave()
	289	* _[spin\|read\|write]_lock_bh()
	290	*/
	291	BUILD_LOCK_OPS(spin, spinlock);
	292	BUILD_LOCK_OPS(read, rwlock);
	293	BUILD_LOCK_OPS(write, rwlock);
	294
	295	#endif /* CONFIG_PREEMPT */
	296
8a25d5de IM	297	#ifdef CONFIG_DEBUG_LOCK_ALLOC
	298
	299	void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
	300	{
	301	preempt_disable();
	302	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
	303	_raw_spin_lock(lock);
	304	}
	305
	306	EXPORT_SYMBOL(_spin_lock_nested);
	307
	308	#endif
	309
1da177e4 LT	310	void __lockfunc _spin_unlock(spinlock_t *lock)
1da177e4 LT	311	{
8a25d5de	312	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	313	_raw_spin_unlock(lock);
	314	preempt_enable();
	315	}
	316	EXPORT_SYMBOL(_spin_unlock);
	317
	318	void __lockfunc _write_unlock(rwlock_t *lock)
	319	{
8a25d5de	320	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	321	_raw_write_unlock(lock);
	322	preempt_enable();
	323	}
	324	EXPORT_SYMBOL(_write_unlock);
	325
	326	void __lockfunc _read_unlock(rwlock_t *lock)
	327	{
8a25d5de	328	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	329	_raw_read_unlock(lock);
	330	preempt_enable();
	331	}
	332	EXPORT_SYMBOL(_read_unlock);
	333
	334	void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
	335	{
8a25d5de	336	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	337	_raw_spin_unlock(lock);
	338	local_irq_restore(flags);
	339	preempt_enable();
	340	}
	341	EXPORT_SYMBOL(_spin_unlock_irqrestore);
	342
	343	void __lockfunc _spin_unlock_irq(spinlock_t *lock)
	344	{
8a25d5de	345	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	346	_raw_spin_unlock(lock);
	347	local_irq_enable();
	348	preempt_enable();
	349	}
	350	EXPORT_SYMBOL(_spin_unlock_irq);
	351
	352	void __lockfunc _spin_unlock_bh(spinlock_t *lock)
	353	{
8a25d5de	354	spin_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	355	_raw_spin_unlock(lock);
10f02d1c	356	preempt_enable_no_resched();
8a25d5de	357	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	358	}
	359	EXPORT_SYMBOL(_spin_unlock_bh);
	360
	361	void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	362	{
8a25d5de	363	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	364	_raw_read_unlock(lock);
	365	local_irq_restore(flags);
	366	preempt_enable();
	367	}
	368	EXPORT_SYMBOL(_read_unlock_irqrestore);
	369
	370	void __lockfunc _read_unlock_irq(rwlock_t *lock)
	371	{
8a25d5de	372	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	373	_raw_read_unlock(lock);
	374	local_irq_enable();
	375	preempt_enable();
	376	}
	377	EXPORT_SYMBOL(_read_unlock_irq);
	378
	379	void __lockfunc _read_unlock_bh(rwlock_t *lock)
	380	{
8a25d5de	381	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	382	_raw_read_unlock(lock);
10f02d1c	383	preempt_enable_no_resched();
8a25d5de	384	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	385	}
	386	EXPORT_SYMBOL(_read_unlock_bh);
	387
	388	void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
	389	{
8a25d5de	390	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	391	_raw_write_unlock(lock);
	392	local_irq_restore(flags);
	393	preempt_enable();
	394	}
	395	EXPORT_SYMBOL(_write_unlock_irqrestore);
	396
	397	void __lockfunc _write_unlock_irq(rwlock_t *lock)
	398	{
8a25d5de	399	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4 LT	400	_raw_write_unlock(lock);
	401	local_irq_enable();
	402	preempt_enable();
	403	}
	404	EXPORT_SYMBOL(_write_unlock_irq);
	405
	406	void __lockfunc _write_unlock_bh(rwlock_t *lock)
	407	{
8a25d5de	408	rwlock_release(&lock->dep_map, 1, _RET_IP_);
1da177e4	409	_raw_write_unlock(lock);
10f02d1c	410	preempt_enable_no_resched();
8a25d5de	411	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	412	}
	413	EXPORT_SYMBOL(_write_unlock_bh);
	414
	415	int __lockfunc _spin_trylock_bh(spinlock_t *lock)
	416	{
	417	local_bh_disable();
	418	preempt_disable();
8a25d5de IM	419	if (_raw_spin_trylock(lock)) {
8a25d5de IM	420	spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1da177e4	421	return 1;
8a25d5de	422	}
1da177e4	423
10f02d1c	424	preempt_enable_no_resched();
8a25d5de	425	local_bh_enable_ip((unsigned long)__builtin_return_address(0));
1da177e4 LT	426	return 0;
	427	}
	428	EXPORT_SYMBOL(_spin_trylock_bh);
	429
	430	int in_lock_functions(unsigned long addr)
	431	{
	432	/* Linker adds these: start and end of __lockfunc functions */
	433	extern char __lock_text_start[], __lock_text_end[];
	434
	435	return addr >= (unsigned long)__lock_text_start
	436	&& addr < (unsigned long)__lock_text_end;
	437	}
	438	EXPORT_SYMBOL(in_lock_functions);