[GitHub/mt8127/android_kernel_alcatel_ttab.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 *	Distribute under GPLv2.
 *
 *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

static inline void _local_bh_enable_ip(unsigned long ip)
{
	WARN_ON_ONCE(in_irq() || irqs_disabled());
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_disable();
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_enable();
#endif
	preempt_check_resched();
}

void local_bh_enable(void)
{
	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
	_local_bh_enable_ip(ip);
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			h->action(h);
			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
#ifdef CONFIG_NO_HZ
	int cpu = smp_processor_id();
	if (idle_cpu(cpu) && !in_interrupt())
		tick_nohz_stop_idle(cpu);
#endif
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(cpu))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick(0);
	rcu_irq_exit();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

void raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action *))
{
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *head;
	struct tasklet_struct **tail;
};

static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);

void __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_vec).tail = t;
	__get_cpu_var(tasklet_vec).tail = &(t->next);
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_hi_vec).tail = t;
	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).head;
	__get_cpu_var(tasklet_vec).head = NULL;
	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_vec).tail = t;
		__get_cpu_var(tasklet_vec).tail = &(t->next);
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).head;
	__get_cpu_var(tasklet_hi_vec).head = NULL;
	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_hi_vec).tail = t;
		__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		per_cpu(tasklet_vec, cpu).tail =
			&per_cpu(tasklet_vec, cpu).head;
		per_cpu(tasklet_hi_vec, cpu).tail =
			&per_cpu(tasklet_hi_vec, cpu).head;
	}

	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
}

static int ksoftirqd(void * __bind_cpu)
{
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			/* If this was the tail element, move the tail ptr */
			if (*i == NULL)
				per_cpu(tasklet_vec, cpu).tail = i;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
		*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
		__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
		per_cpu(tasklet_vec, cpu).head = NULL;
		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	}
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
		*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
		__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
		per_cpu(tasklet_hi_vec, cpu).head = NULL;
		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	}
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

__init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
b10db7f0 PM	6	* Distribute under GPLv2.
	7	*
	8	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
1da177e4 LT	9	*/
	10
	11	#include <linux/module.h>
	12	#include <linux/kernel_stat.h>
	13	#include <linux/interrupt.h>
	14	#include <linux/init.h>
	15	#include <linux/mm.h>
	16	#include <linux/notifier.h>
	17	#include <linux/percpu.h>
	18	#include <linux/cpu.h>
83144186	19	#include <linux/freezer.h>
1da177e4 LT	20	#include <linux/kthread.h>
1da177e4 LT	21	#include <linux/rcupdate.h>
78eef01b	22	#include <linux/smp.h>
79bf2bb3	23	#include <linux/tick.h>
1da177e4 LT	24
	25	#include <asm/irq.h>
	26	/*
	27	- No shared variables, all the data are CPU local.
	28	- If a softirq needs serialization, let it serialize itself
	29	by its own spinlocks.
	30	- Even if softirq is serialized, only local cpu is marked for
	31	execution. Hence, we get something sort of weak cpu binding.
	32	Though it is still not clear, will it result in better locality
	33	or will not.
	34
	35	Examples:
	36	- NET RX softirq. It is multithreaded and does not require
	37	any global serialization.
	38	- NET TX softirq. It kicks software netdevice queues, hence
	39	it is logically serialized per device, but this serialization
	40	is invisible to common code.
	41	- Tasklets: serialized wrt itself.
	42	*/
	43
	44	#ifndef __ARCH_IRQ_STAT
	45	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	46	EXPORT_SYMBOL(irq_stat);
	47	#endif
	48
	49	static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
	50
	51	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	52
	53	/*
	54	* we cannot loop indefinitely here to avoid userspace starvation,
	55	* but we also don't want to introduce a worst case 1/HZ latency
	56	* to the pending events, so lets the scheduler to balance
	57	* the softirq load for us.
	58	*/
	59	static inline void wakeup_softirqd(void)
	60	{
	61	/* Interrupts are disabled: no need to stop preemption */
	62	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	63
	64	if (tsk && tsk->state != TASK_RUNNING)
	65	wake_up_process(tsk);
	66	}
	67
de30a2b3 IM	68	/*
	69	* This one is for softirq.c-internal use,
	70	* where hardirqs are disabled legitimately:
	71	*/
3c829c36	72	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	73	static void __local_bh_disable(unsigned long ip)
	74	{
	75	unsigned long flags;
	76
	77	WARN_ON_ONCE(in_irq());
	78
	79	raw_local_irq_save(flags);
	80	add_preempt_count(SOFTIRQ_OFFSET);
	81	/*
	82	* Were softirqs turned off above:
	83	*/
	84	if (softirq_count() == SOFTIRQ_OFFSET)
	85	trace_softirqs_off(ip);
	86	raw_local_irq_restore(flags);
	87	}
3c829c36 TC	88	#else /* !CONFIG_TRACE_IRQFLAGS */
	89	static inline void __local_bh_disable(unsigned long ip)
	90	{
	91	add_preempt_count(SOFTIRQ_OFFSET);
	92	barrier();
	93	}
	94	#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 IM	95
	96	void local_bh_disable(void)
	97	{
	98	__local_bh_disable((unsigned long)__builtin_return_address(0));
	99	}
	100
	101	EXPORT_SYMBOL(local_bh_disable);
	102
	103	void __local_bh_enable(void)
	104	{
	105	WARN_ON_ONCE(in_irq());
	106
	107	/*
	108	* softirqs should never be enabled by __local_bh_enable(),
	109	* it always nests inside local_bh_enable() sections:
	110	*/
	111	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	112
	113	sub_preempt_count(SOFTIRQ_OFFSET);
	114	}
	115	EXPORT_SYMBOL_GPL(__local_bh_enable);
	116
	117	/*
	118	* Special-case - softirqs can safely be enabled in
	119	* cond_resched_softirq(), or by __do_softirq(),
	120	* without processing still-pending softirqs:
	121	*/
	122	void _local_bh_enable(void)
	123	{
	124	WARN_ON_ONCE(in_irq());
	125	WARN_ON_ONCE(!irqs_disabled());
	126
	127	if (softirq_count() == SOFTIRQ_OFFSET)
	128	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	129	sub_preempt_count(SOFTIRQ_OFFSET);
	130	}
	131
	132	EXPORT_SYMBOL(_local_bh_enable);
	133
0f476b6d	134	static inline void _local_bh_enable_ip(unsigned long ip)
de30a2b3	135	{
0f476b6d	136	WARN_ON_ONCE(in_irq() \|\| irqs_disabled());
3c829c36	137	#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d	138	local_irq_disable();
3c829c36	139	#endif
de30a2b3 IM	140	/*
	141	* Are softirqs going to be turned on now:
	142	*/
	143	if (softirq_count() == SOFTIRQ_OFFSET)
0f476b6d	144	trace_softirqs_on(ip);
de30a2b3 IM	145	/*
	146	* Keep preemption disabled until we are done with
	147	* softirq processing:
	148	*/
	149	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	150
	151	if (unlikely(!in_interrupt() && local_softirq_pending()))
	152	do_softirq();
	153
	154	dec_preempt_count();
3c829c36	155	#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d	156	local_irq_enable();
3c829c36	157	#endif
de30a2b3 IM	158	preempt_check_resched();
de30a2b3 IM	159	}
0f476b6d JB	160
	161	void local_bh_enable(void)
	162	{
	163	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	164	}
de30a2b3 IM	165	EXPORT_SYMBOL(local_bh_enable);
	166
	167	void local_bh_enable_ip(unsigned long ip)
	168	{
0f476b6d	169	_local_bh_enable_ip(ip);
de30a2b3 IM	170	}
	171	EXPORT_SYMBOL(local_bh_enable_ip);
	172
1da177e4 LT	173	/*
	174	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	175	* and we fall back to softirqd after that.
	176	*
	177	* This number has been established via experimentation.
	178	* The two things to balance is latency against fairness -
	179	* we want to handle softirqs as soon as possible, but they
	180	* should not be able to lock up the box.
	181	*/
	182	#define MAX_SOFTIRQ_RESTART 10
	183
	184	asmlinkage void __do_softirq(void)
	185	{
	186	struct softirq_action *h;
	187	__u32 pending;
	188	int max_restart = MAX_SOFTIRQ_RESTART;
	189	int cpu;
	190
	191	pending = local_softirq_pending();
829035fd PM	192	account_system_vtime(current);
829035fd PM	193
de30a2b3 IM	194	__local_bh_disable((unsigned long)__builtin_return_address(0));
de30a2b3 IM	195	trace_softirq_enter();
1da177e4	196
1da177e4 LT	197	cpu = smp_processor_id();
	198	restart:
	199	/* Reset the pending bitmask before enabling irqs */
3f74478b	200	set_softirq_pending(0);
1da177e4	201
c70f5d66	202	local_irq_enable();
1da177e4 LT	203
	204	h = softirq_vec;
	205
	206	do {
	207	if (pending & 1) {
	208	h->action(h);
	209	rcu_bh_qsctr_inc(cpu);
	210	}
	211	h++;
	212	pending >>= 1;
	213	} while (pending);
	214
c70f5d66	215	local_irq_disable();
1da177e4 LT	216
	217	pending = local_softirq_pending();
	218	if (pending && --max_restart)
	219	goto restart;
	220
	221	if (pending)
	222	wakeup_softirqd();
	223
de30a2b3	224	trace_softirq_exit();
829035fd PM	225
829035fd PM	226	account_system_vtime(current);
de30a2b3	227	_local_bh_enable();
1da177e4 LT	228	}
	229
	230	#ifndef __ARCH_HAS_DO_SOFTIRQ
	231
	232	asmlinkage void do_softirq(void)
	233	{
	234	__u32 pending;
	235	unsigned long flags;
	236
	237	if (in_interrupt())
	238	return;
	239
	240	local_irq_save(flags);
	241
	242	pending = local_softirq_pending();
	243
	244	if (pending)
	245	__do_softirq();
	246
	247	local_irq_restore(flags);
	248	}
	249
1da177e4 LT	250	#endif
1da177e4 LT	251
dde4b2b5 IM	252	/*
	253	* Enter an interrupt context.
	254	*/
	255	void irq_enter(void)
	256	{
6378ddb5 VP	257	#ifdef CONFIG_NO_HZ
	258	int cpu = smp_processor_id();
	259	if (idle_cpu(cpu) && !in_interrupt())
	260	tick_nohz_stop_idle(cpu);
	261	#endif
79bf2bb3 TG	262	__irq_enter();
79bf2bb3 TG	263	#ifdef CONFIG_NO_HZ
6378ddb5	264	if (idle_cpu(cpu))
79bf2bb3 TG	265	tick_nohz_update_jiffies();
79bf2bb3 TG	266	#endif
dde4b2b5 IM	267	}
dde4b2b5 IM	268
1da177e4 LT	269	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	270	# define invoke_softirq() __do_softirq()
	271	#else
	272	# define invoke_softirq() do_softirq()
	273	#endif
	274
	275	/*
	276	* Exit an interrupt context. Process softirqs if needed and possible:
	277	*/
	278	void irq_exit(void)
	279	{
	280	account_system_vtime(current);
de30a2b3	281	trace_hardirq_exit();
1da177e4 LT	282	sub_preempt_count(IRQ_EXIT_OFFSET);
	283	if (!in_interrupt() && local_softirq_pending())
	284	invoke_softirq();
79bf2bb3 TG	285
	286	#ifdef CONFIG_NO_HZ
	287	/* Make sure that timer wheel updates are propagated */
	288	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
b8f8c3cf	289	tick_nohz_stop_sched_tick(0);
2232c2d8	290	rcu_irq_exit();
79bf2bb3	291	#endif
1da177e4 LT	292	preempt_enable_no_resched();
	293	}
	294
	295	/*
	296	* This function must run with irqs disabled!
	297	*/
7ad5b3a5	298	inline void raise_softirq_irqoff(unsigned int nr)
1da177e4 LT	299	{
	300	__raise_softirq_irqoff(nr);
	301
	302	/*
	303	* If we're in an interrupt or softirq, we're done
	304	* (this also catches softirq-disabled code). We will
	305	* actually run the softirq once we return from
	306	* the irq or softirq.
	307	*
	308	* Otherwise we wake up ksoftirqd to make sure we
	309	* schedule the softirq soon.
	310	*/
	311	if (!in_interrupt())
	312	wakeup_softirqd();
	313	}
	314
7ad5b3a5	315	void raise_softirq(unsigned int nr)
1da177e4 LT	316	{
	317	unsigned long flags;
	318
	319	local_irq_save(flags);
	320	raise_softirq_irqoff(nr);
	321	local_irq_restore(flags);
	322	}
	323
962cf36c	324	void open_softirq(int nr, void (action)(struct softirq_action ))
1da177e4	325	{
1da177e4 LT	326	softirq_vec[nr].action = action;
	327	}
	328
1da177e4 LT	329	/* Tasklets */
	330	struct tasklet_head
	331	{
48f20a9a OJ	332	struct tasklet_struct *head;
48f20a9a OJ	333	struct tasklet_struct **tail;
1da177e4 LT	334	};
1da177e4 LT	335
4620b49f VN	336	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
4620b49f VN	337	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
1da177e4	338
7ad5b3a5	339	void __tasklet_schedule(struct tasklet_struct *t)
1da177e4 LT	340	{
	341	unsigned long flags;
	342
	343	local_irq_save(flags);
48f20a9a OJ	344	t->next = NULL;
	345	*__get_cpu_var(tasklet_vec).tail = t;
	346	__get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4 LT	347	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	348	local_irq_restore(flags);
	349	}
	350
	351	EXPORT_SYMBOL(__tasklet_schedule);
	352
7ad5b3a5	353	void __tasklet_hi_schedule(struct tasklet_struct *t)
1da177e4 LT	354	{
	355	unsigned long flags;
	356
	357	local_irq_save(flags);
48f20a9a OJ	358	t->next = NULL;
	359	*__get_cpu_var(tasklet_hi_vec).tail = t;
	360	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4 LT	361	raise_softirq_irqoff(HI_SOFTIRQ);
	362	local_irq_restore(flags);
	363	}
	364
	365	EXPORT_SYMBOL(__tasklet_hi_schedule);
	366
	367	static void tasklet_action(struct softirq_action *a)
	368	{
	369	struct tasklet_struct *list;
	370
	371	local_irq_disable();
48f20a9a OJ	372	list = __get_cpu_var(tasklet_vec).head;
	373	__get_cpu_var(tasklet_vec).head = NULL;
	374	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
1da177e4 LT	375	local_irq_enable();
	376
	377	while (list) {
	378	struct tasklet_struct *t = list;
	379
	380	list = list->next;
	381
	382	if (tasklet_trylock(t)) {
	383	if (!atomic_read(&t->count)) {
	384	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	385	BUG();
	386	t->func(t->data);
	387	tasklet_unlock(t);
	388	continue;
	389	}
	390	tasklet_unlock(t);
	391	}
	392
	393	local_irq_disable();
48f20a9a OJ	394	t->next = NULL;
	395	*__get_cpu_var(tasklet_vec).tail = t;
	396	__get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4 LT	397	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
	398	local_irq_enable();
	399	}
	400	}
	401
	402	static void tasklet_hi_action(struct softirq_action *a)
	403	{
	404	struct tasklet_struct *list;
	405
	406	local_irq_disable();
48f20a9a OJ	407	list = __get_cpu_var(tasklet_hi_vec).head;
	408	__get_cpu_var(tasklet_hi_vec).head = NULL;
	409	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
1da177e4 LT	410	local_irq_enable();
	411
	412	while (list) {
	413	struct tasklet_struct *t = list;
	414
	415	list = list->next;
	416
	417	if (tasklet_trylock(t)) {
	418	if (!atomic_read(&t->count)) {
	419	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	420	BUG();
	421	t->func(t->data);
	422	tasklet_unlock(t);
	423	continue;
	424	}
	425	tasklet_unlock(t);
	426	}
	427
	428	local_irq_disable();
48f20a9a OJ	429	t->next = NULL;
	430	*__get_cpu_var(tasklet_hi_vec).tail = t;
	431	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4 LT	432	__raise_softirq_irqoff(HI_SOFTIRQ);
	433	local_irq_enable();
	434	}
	435	}
	436
	437
	438	void tasklet_init(struct tasklet_struct *t,
	439	void (*func)(unsigned long), unsigned long data)
	440	{
	441	t->next = NULL;
	442	t->state = 0;
	443	atomic_set(&t->count, 0);
	444	t->func = func;
	445	t->data = data;
	446	}
	447
	448	EXPORT_SYMBOL(tasklet_init);
	449
	450	void tasklet_kill(struct tasklet_struct *t)
	451	{
	452	if (in_interrupt())
	453	printk("Attempt to kill tasklet from interrupt\n");
	454
	455	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
	456	do
	457	yield();
	458	while (test_bit(TASKLET_STATE_SCHED, &t->state));
	459	}
	460	tasklet_unlock_wait(t);
	461	clear_bit(TASKLET_STATE_SCHED, &t->state);
	462	}
	463
	464	EXPORT_SYMBOL(tasklet_kill);
	465
	466	void __init softirq_init(void)
	467	{
48f20a9a OJ	468	int cpu;
	469
	470	for_each_possible_cpu(cpu) {
	471	per_cpu(tasklet_vec, cpu).tail =
	472	&per_cpu(tasklet_vec, cpu).head;
	473	per_cpu(tasklet_hi_vec, cpu).tail =
	474	&per_cpu(tasklet_hi_vec, cpu).head;
	475	}
	476
962cf36c CM	477	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
962cf36c CM	478	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1da177e4 LT	479	}
	480
	481	static int ksoftirqd(void * __bind_cpu)
	482	{
1da177e4 LT	483	set_current_state(TASK_INTERRUPTIBLE);
	484
	485	while (!kthread_should_stop()) {
	486	preempt_disable();
	487	if (!local_softirq_pending()) {
	488	preempt_enable_no_resched();
	489	schedule();
	490	preempt_disable();
	491	}
	492
	493	__set_current_state(TASK_RUNNING);
	494
	495	while (local_softirq_pending()) {
	496	/* Preempt disable stops cpu going offline.
	497	If already offline, we'll be on wrong CPU:
	498	don't process */
	499	if (cpu_is_offline((long)__bind_cpu))
	500	goto wait_to_die;
	501	do_softirq();
	502	preempt_enable_no_resched();
	503	cond_resched();
	504	preempt_disable();
	505	}
	506	preempt_enable();
	507	set_current_state(TASK_INTERRUPTIBLE);
	508	}
	509	__set_current_state(TASK_RUNNING);
	510	return 0;
	511
	512	wait_to_die:
	513	preempt_enable();
	514	/* Wait for kthread_stop */
	515	set_current_state(TASK_INTERRUPTIBLE);
	516	while (!kthread_should_stop()) {
	517	schedule();
	518	set_current_state(TASK_INTERRUPTIBLE);
	519	}
	520	__set_current_state(TASK_RUNNING);
	521	return 0;
	522	}
	523
	524	#ifdef CONFIG_HOTPLUG_CPU
	525	/*
	526	* tasklet_kill_immediate is called to remove a tasklet which can already be
	527	* scheduled for execution on @cpu.
	528	*
	529	* Unlike tasklet_kill, this function removes the tasklet
	530	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	531	*
	532	* When this function is called, @cpu must be in the CPU_DEAD state.
	533	*/
	534	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	535	{
	536	struct tasklet_struct **i;
	537
	538	BUG_ON(cpu_online(cpu));
	539	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	540
	541	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	542	return;
	543
	544	/* CPU is dead, so no lock needed. */
48f20a9a	545	for (i = &per_cpu(tasklet_vec, cpu).head; i; i = &(i)->next) {
1da177e4 LT	546	if (*i == t) {
1da177e4 LT	547	*i = t->next;
48f20a9a OJ	548	/* If this was the tail element, move the tail ptr */
	549	if (*i == NULL)
	550	per_cpu(tasklet_vec, cpu).tail = i;
1da177e4 LT	551	return;
	552	}
	553	}
	554	BUG();
	555	}
	556
	557	static void takeover_tasklets(unsigned int cpu)
	558	{
1da177e4 LT	559	/* CPU is dead, so no lock needed. */
	560	local_irq_disable();
	561
	562	/* Find end, append list for that CPU. */
e5e41723 CB	563	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
	564	*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
	565	__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
	566	per_cpu(tasklet_vec, cpu).head = NULL;
	567	per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	568	}
1da177e4 LT	569	raise_softirq_irqoff(TASKLET_SOFTIRQ);
1da177e4 LT	570
e5e41723 CB	571	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
	572	*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
	573	__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
	574	per_cpu(tasklet_hi_vec, cpu).head = NULL;
	575	per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	576	}
1da177e4 LT	577	raise_softirq_irqoff(HI_SOFTIRQ);
	578
	579	local_irq_enable();
	580	}
	581	#endif /* CONFIG_HOTPLUG_CPU */
	582
8c78f307	583	static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4 LT	584	unsigned long action,
	585	void *hcpu)
	586	{
	587	int hotcpu = (unsigned long)hcpu;
	588	struct task_struct *p;
	589
	590	switch (action) {
	591	case CPU_UP_PREPARE:
8bb78442	592	case CPU_UP_PREPARE_FROZEN:
1da177e4 LT	593	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	594	if (IS_ERR(p)) {
	595	printk("ksoftirqd for %i failed\n", hotcpu);
	596	return NOTIFY_BAD;
	597	}
	598	kthread_bind(p, hotcpu);
	599	per_cpu(ksoftirqd, hotcpu) = p;
	600	break;
	601	case CPU_ONLINE:
8bb78442	602	case CPU_ONLINE_FROZEN:
1da177e4 LT	603	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	604	break;
	605	#ifdef CONFIG_HOTPLUG_CPU
	606	case CPU_UP_CANCELED:
8bb78442	607	case CPU_UP_CANCELED_FROZEN:
fc75cdfa HC	608	if (!per_cpu(ksoftirqd, hotcpu))
fc75cdfa HC	609	break;
1da177e4	610	/* Unbind so it can run. Fall thru. */
a4c4af7c HC	611	kthread_bind(per_cpu(ksoftirqd, hotcpu),
a4c4af7c HC	612	any_online_cpu(cpu_online_map));
1da177e4	613	case CPU_DEAD:
1c6b4aa9 ST	614	case CPU_DEAD_FROZEN: {
	615	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	616
1da177e4 LT	617	p = per_cpu(ksoftirqd, hotcpu);
1da177e4 LT	618	per_cpu(ksoftirqd, hotcpu) = NULL;
961ccddd	619	sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
1da177e4 LT	620	kthread_stop(p);
	621	takeover_tasklets(hotcpu);
	622	break;
1c6b4aa9	623	}
1da177e4 LT	624	#endif /* CONFIG_HOTPLUG_CPU */
	625	}
	626	return NOTIFY_OK;
	627	}
	628
8c78f307	629	static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4 LT	630	.notifier_call = cpu_callback
	631	};
	632
	633	__init int spawn_ksoftirqd(void)
	634	{
	635	void cpu = (void )(long)smp_processor_id();
07dccf33 AM	636	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	637
	638	BUG_ON(err == NOTIFY_BAD);
1da177e4 LT	639	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	640	register_cpu_notifier(&cpu_nfb);
	641	return 0;
	642	}
78eef01b AM	643
	644	#ifdef CONFIG_SMP
	645	/*
	646	* Call a function on all processors
	647	*/
15c8b6c1	648	int on_each_cpu(void (func) (void info), void *info, int wait)
78eef01b AM	649	{
	650	int ret = 0;
	651
	652	preempt_disable();
8691e5a8	653	ret = smp_call_function(func, info, wait);
78eef01b AM	654	local_irq_disable();
	655	func(info);
	656	local_irq_enable();
	657	preempt_enable();
	658	return ret;
	659	}
	660	EXPORT_SYMBOL(on_each_cpu);
	661	#endif