[GitHub/mt8127/android_kernel_alcatel_ttab.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

void local_bh_enable(void)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());
#endif
	WARN_ON_ONCE(irqs_disabled());

#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			h->action(h);
			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(smp_processor_id()))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline fastcall void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

void fastcall raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
{
	softirq_vec[nr].data = data;
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *list;
};

/* Some compilers disobey section attribute on statics when not
   initialized -- RR */
static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };

void fastcall __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = __get_cpu_var(tasklet_vec).list;
	__get_cpu_var(tasklet_vec).list = t;
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = __get_cpu_var(tasklet_hi_vec).list;
	__get_cpu_var(tasklet_hi_vec).list = t;
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).list;
	__get_cpu_var(tasklet_vec).list = NULL;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = __get_cpu_var(tasklet_vec).list;
		__get_cpu_var(tasklet_vec).list = t;
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).list;
	__get_cpu_var(tasklet_hi_vec).list = NULL;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = __get_cpu_var(tasklet_hi_vec).list;
		__get_cpu_var(tasklet_hi_vec).list = t;
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
}

static int ksoftirqd(void * __bind_cpu)
{
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	struct tasklet_struct **i;

	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
	*i = per_cpu(tasklet_vec, cpu).list;
	per_cpu(tasklet_vec, cpu).list = NULL;
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
	*i = per_cpu(tasklet_hi_vec, cpu).list;
	per_cpu(tasklet_hi_vec, cpu).list = NULL;
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

__init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, retry, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
	6	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
	7	*/
	8
	9	#include <linux/module.h>
	10	#include <linux/kernel_stat.h>
	11	#include <linux/interrupt.h>
	12	#include <linux/init.h>
	13	#include <linux/mm.h>
	14	#include <linux/notifier.h>
	15	#include <linux/percpu.h>
	16	#include <linux/cpu.h>
83144186	17	#include <linux/freezer.h>
1da177e4 LT	18	#include <linux/kthread.h>
1da177e4 LT	19	#include <linux/rcupdate.h>
78eef01b	20	#include <linux/smp.h>
79bf2bb3	21	#include <linux/tick.h>
1da177e4 LT	22
	23	#include <asm/irq.h>
	24	/*
	25	- No shared variables, all the data are CPU local.
	26	- If a softirq needs serialization, let it serialize itself
	27	by its own spinlocks.
	28	- Even if softirq is serialized, only local cpu is marked for
	29	execution. Hence, we get something sort of weak cpu binding.
	30	Though it is still not clear, will it result in better locality
	31	or will not.
	32
	33	Examples:
	34	- NET RX softirq. It is multithreaded and does not require
	35	any global serialization.
	36	- NET TX softirq. It kicks software netdevice queues, hence
	37	it is logically serialized per device, but this serialization
	38	is invisible to common code.
	39	- Tasklets: serialized wrt itself.
	40	*/
	41
	42	#ifndef __ARCH_IRQ_STAT
	43	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	44	EXPORT_SYMBOL(irq_stat);
	45	#endif
	46
	47	static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
	48
	49	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	50
	51	/*
	52	* we cannot loop indefinitely here to avoid userspace starvation,
	53	* but we also don't want to introduce a worst case 1/HZ latency
	54	* to the pending events, so lets the scheduler to balance
	55	* the softirq load for us.
	56	*/
	57	static inline void wakeup_softirqd(void)
	58	{
	59	/* Interrupts are disabled: no need to stop preemption */
	60	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	61
	62	if (tsk && tsk->state != TASK_RUNNING)
	63	wake_up_process(tsk);
	64	}
	65
de30a2b3 IM	66	/*
	67	* This one is for softirq.c-internal use,
	68	* where hardirqs are disabled legitimately:
	69	*/
3c829c36	70	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	71	static void __local_bh_disable(unsigned long ip)
	72	{
	73	unsigned long flags;
	74
	75	WARN_ON_ONCE(in_irq());
	76
	77	raw_local_irq_save(flags);
	78	add_preempt_count(SOFTIRQ_OFFSET);
	79	/*
	80	* Were softirqs turned off above:
	81	*/
	82	if (softirq_count() == SOFTIRQ_OFFSET)
	83	trace_softirqs_off(ip);
	84	raw_local_irq_restore(flags);
	85	}
3c829c36 TC	86	#else /* !CONFIG_TRACE_IRQFLAGS */
	87	static inline void __local_bh_disable(unsigned long ip)
	88	{
	89	add_preempt_count(SOFTIRQ_OFFSET);
	90	barrier();
	91	}
	92	#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 IM	93
	94	void local_bh_disable(void)
	95	{
	96	__local_bh_disable((unsigned long)__builtin_return_address(0));
	97	}
	98
	99	EXPORT_SYMBOL(local_bh_disable);
	100
	101	void __local_bh_enable(void)
	102	{
	103	WARN_ON_ONCE(in_irq());
	104
	105	/*
	106	* softirqs should never be enabled by __local_bh_enable(),
	107	* it always nests inside local_bh_enable() sections:
	108	*/
	109	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	110
	111	sub_preempt_count(SOFTIRQ_OFFSET);
	112	}
	113	EXPORT_SYMBOL_GPL(__local_bh_enable);
	114
	115	/*
	116	* Special-case - softirqs can safely be enabled in
	117	* cond_resched_softirq(), or by __do_softirq(),
	118	* without processing still-pending softirqs:
	119	*/
	120	void _local_bh_enable(void)
	121	{
	122	WARN_ON_ONCE(in_irq());
	123	WARN_ON_ONCE(!irqs_disabled());
	124
	125	if (softirq_count() == SOFTIRQ_OFFSET)
	126	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	127	sub_preempt_count(SOFTIRQ_OFFSET);
	128	}
	129
	130	EXPORT_SYMBOL(_local_bh_enable);
	131
	132	void local_bh_enable(void)
	133	{
3c829c36	134	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	135	unsigned long flags;
	136
	137	WARN_ON_ONCE(in_irq());
3c829c36	138	#endif
de30a2b3 IM	139	WARN_ON_ONCE(irqs_disabled());
de30a2b3 IM	140
3c829c36	141	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	142	local_irq_save(flags);
3c829c36	143	#endif
de30a2b3 IM	144	/*
	145	* Are softirqs going to be turned on now:
	146	*/
	147	if (softirq_count() == SOFTIRQ_OFFSET)
	148	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	149	/*
	150	* Keep preemption disabled until we are done with
	151	* softirq processing:
	152	*/
	153	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	154
	155	if (unlikely(!in_interrupt() && local_softirq_pending()))
	156	do_softirq();
	157
	158	dec_preempt_count();
3c829c36	159	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	160	local_irq_restore(flags);
3c829c36	161	#endif
de30a2b3 IM	162	preempt_check_resched();
	163	}
	164	EXPORT_SYMBOL(local_bh_enable);
	165
	166	void local_bh_enable_ip(unsigned long ip)
	167	{
3c829c36	168	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	169	unsigned long flags;
	170
	171	WARN_ON_ONCE(in_irq());
	172
	173	local_irq_save(flags);
3c829c36	174	#endif
de30a2b3 IM	175	/*
	176	* Are softirqs going to be turned on now:
	177	*/
	178	if (softirq_count() == SOFTIRQ_OFFSET)
	179	trace_softirqs_on(ip);
	180	/*
	181	* Keep preemption disabled until we are done with
	182	* softirq processing:
	183	*/
	184	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	185
	186	if (unlikely(!in_interrupt() && local_softirq_pending()))
	187	do_softirq();
	188
	189	dec_preempt_count();
3c829c36	190	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	191	local_irq_restore(flags);
3c829c36	192	#endif
de30a2b3 IM	193	preempt_check_resched();
	194	}
	195	EXPORT_SYMBOL(local_bh_enable_ip);
	196
1da177e4 LT	197	/*
	198	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	199	* and we fall back to softirqd after that.
	200	*
	201	* This number has been established via experimentation.
	202	* The two things to balance is latency against fairness -
	203	* we want to handle softirqs as soon as possible, but they
	204	* should not be able to lock up the box.
	205	*/
	206	#define MAX_SOFTIRQ_RESTART 10
	207
	208	asmlinkage void __do_softirq(void)
	209	{
	210	struct softirq_action *h;
	211	__u32 pending;
	212	int max_restart = MAX_SOFTIRQ_RESTART;
	213	int cpu;
	214
	215	pending = local_softirq_pending();
829035fd PM	216	account_system_vtime(current);
829035fd PM	217
de30a2b3 IM	218	__local_bh_disable((unsigned long)__builtin_return_address(0));
de30a2b3 IM	219	trace_softirq_enter();
1da177e4	220
1da177e4 LT	221	cpu = smp_processor_id();
	222	restart:
	223	/* Reset the pending bitmask before enabling irqs */
3f74478b	224	set_softirq_pending(0);
1da177e4	225
c70f5d66	226	local_irq_enable();
1da177e4 LT	227
	228	h = softirq_vec;
	229
	230	do {
	231	if (pending & 1) {
	232	h->action(h);
	233	rcu_bh_qsctr_inc(cpu);
	234	}
	235	h++;
	236	pending >>= 1;
	237	} while (pending);
	238
c70f5d66	239	local_irq_disable();
1da177e4 LT	240
	241	pending = local_softirq_pending();
	242	if (pending && --max_restart)
	243	goto restart;
	244
	245	if (pending)
	246	wakeup_softirqd();
	247
de30a2b3	248	trace_softirq_exit();
829035fd PM	249
829035fd PM	250	account_system_vtime(current);
de30a2b3	251	_local_bh_enable();
1da177e4 LT	252	}
	253
	254	#ifndef __ARCH_HAS_DO_SOFTIRQ
	255
	256	asmlinkage void do_softirq(void)
	257	{
	258	__u32 pending;
	259	unsigned long flags;
	260
	261	if (in_interrupt())
	262	return;
	263
	264	local_irq_save(flags);
	265
	266	pending = local_softirq_pending();
	267
	268	if (pending)
	269	__do_softirq();
	270
	271	local_irq_restore(flags);
	272	}
	273
1da177e4 LT	274	#endif
1da177e4 LT	275
dde4b2b5 IM	276	/*
	277	* Enter an interrupt context.
	278	*/
	279	void irq_enter(void)
	280	{
79bf2bb3 TG	281	__irq_enter();
	282	#ifdef CONFIG_NO_HZ
	283	if (idle_cpu(smp_processor_id()))
	284	tick_nohz_update_jiffies();
	285	#endif
dde4b2b5 IM	286	}
dde4b2b5 IM	287
1da177e4 LT	288	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	289	# define invoke_softirq() __do_softirq()
	290	#else
	291	# define invoke_softirq() do_softirq()
	292	#endif
	293
	294	/*
	295	* Exit an interrupt context. Process softirqs if needed and possible:
	296	*/
	297	void irq_exit(void)
	298	{
	299	account_system_vtime(current);
de30a2b3	300	trace_hardirq_exit();
1da177e4 LT	301	sub_preempt_count(IRQ_EXIT_OFFSET);
	302	if (!in_interrupt() && local_softirq_pending())
	303	invoke_softirq();
79bf2bb3 TG	304
	305	#ifdef CONFIG_NO_HZ
	306	/* Make sure that timer wheel updates are propagated */
	307	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
	308	tick_nohz_stop_sched_tick();
	309	#endif
1da177e4 LT	310	preempt_enable_no_resched();
	311	}
	312
	313	/*
	314	* This function must run with irqs disabled!
	315	*/
	316	inline fastcall void raise_softirq_irqoff(unsigned int nr)
	317	{
	318	__raise_softirq_irqoff(nr);
	319
	320	/*
	321	* If we're in an interrupt or softirq, we're done
	322	* (this also catches softirq-disabled code). We will
	323	* actually run the softirq once we return from
	324	* the irq or softirq.
	325	*
	326	* Otherwise we wake up ksoftirqd to make sure we
	327	* schedule the softirq soon.
	328	*/
	329	if (!in_interrupt())
	330	wakeup_softirqd();
	331	}
	332
1da177e4 LT	333	void fastcall raise_softirq(unsigned int nr)
	334	{
	335	unsigned long flags;
	336
	337	local_irq_save(flags);
	338	raise_softirq_irqoff(nr);
	339	local_irq_restore(flags);
	340	}
	341
	342	void open_softirq(int nr, void (action)(struct softirq_action), void *data)
	343	{
	344	softirq_vec[nr].data = data;
	345	softirq_vec[nr].action = action;
	346	}
	347
1da177e4 LT	348	/* Tasklets */
	349	struct tasklet_head
	350	{
	351	struct tasklet_struct *list;
	352	};
	353
	354	/* Some compilers disobey section attribute on statics when not
	355	initialized -- RR */
	356	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
	357	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
	358
	359	void fastcall __tasklet_schedule(struct tasklet_struct *t)
	360	{
	361	unsigned long flags;
	362
	363	local_irq_save(flags);
	364	t->next = __get_cpu_var(tasklet_vec).list;
	365	__get_cpu_var(tasklet_vec).list = t;
	366	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	367	local_irq_restore(flags);
	368	}
	369
	370	EXPORT_SYMBOL(__tasklet_schedule);
	371
	372	void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
	373	{
	374	unsigned long flags;
	375
	376	local_irq_save(flags);
	377	t->next = __get_cpu_var(tasklet_hi_vec).list;
	378	__get_cpu_var(tasklet_hi_vec).list = t;
	379	raise_softirq_irqoff(HI_SOFTIRQ);
	380	local_irq_restore(flags);
	381	}
	382
	383	EXPORT_SYMBOL(__tasklet_hi_schedule);
	384
	385	static void tasklet_action(struct softirq_action *a)
	386	{
	387	struct tasklet_struct *list;
	388
	389	local_irq_disable();
	390	list = __get_cpu_var(tasklet_vec).list;
	391	__get_cpu_var(tasklet_vec).list = NULL;
	392	local_irq_enable();
	393
	394	while (list) {
	395	struct tasklet_struct *t = list;
	396
	397	list = list->next;
	398
	399	if (tasklet_trylock(t)) {
	400	if (!atomic_read(&t->count)) {
	401	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	402	BUG();
	403	t->func(t->data);
	404	tasklet_unlock(t);
	405	continue;
	406	}
	407	tasklet_unlock(t);
	408	}
	409
	410	local_irq_disable();
	411	t->next = __get_cpu_var(tasklet_vec).list;
412	__get_cpu_var(tasklet_vec).list = t;
413	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
414	local_irq_enable();
415	}
416	}
417
418	static void tasklet_hi_action(struct softirq_action *a)
419	{
420	struct tasklet_struct *list;
421
422	local_irq_disable();
423	list = __get_cpu_var(tasklet_hi_vec).list;
424	__get_cpu_var(tasklet_hi_vec).list = NULL;
425	local_irq_enable();
426
427	while (list) {
428	struct tasklet_struct *t = list;
429
430	list = list->next;
431
432	if (tasklet_trylock(t)) {
433	if (!atomic_read(&t->count)) {
434	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
435	BUG();
436	t->func(t->data);
437	tasklet_unlock(t);
438	continue;
439	}
440	tasklet_unlock(t);
441	}
442
443	local_irq_disable();
444	t->next = __get_cpu_var(tasklet_hi_vec).list;
445	__get_cpu_var(tasklet_hi_vec).list = t;
446	__raise_softirq_irqoff(HI_SOFTIRQ);
447	local_irq_enable();
448	}
449	}
450
451
452	void tasklet_init(struct tasklet_struct *t,
453	void (*func)(unsigned long), unsigned long data)
454	{
455	t->next = NULL;
456	t->state = 0;
457	atomic_set(&t->count, 0);
458	t->func = func;
459	t->data = data;
460	}
461
462	EXPORT_SYMBOL(tasklet_init);
463
464	void tasklet_kill(struct tasklet_struct *t)
465	{
466	if (in_interrupt())
467	printk("Attempt to kill tasklet from interrupt\n");
468
469	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
470	do
471	yield();
472	while (test_bit(TASKLET_STATE_SCHED, &t->state));
473	}
474	tasklet_unlock_wait(t);
475	clear_bit(TASKLET_STATE_SCHED, &t->state);
476	}
477
478	EXPORT_SYMBOL(tasklet_kill);
479
480	void __init softirq_init(void)
481	{
482	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
483	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
484	}
485
486	static int ksoftirqd(void * __bind_cpu)
487	{
1da177e4 LT	488	set_current_state(TASK_INTERRUPTIBLE);
	489
	490	while (!kthread_should_stop()) {
	491	preempt_disable();
	492	if (!local_softirq_pending()) {
	493	preempt_enable_no_resched();
	494	schedule();
	495	preempt_disable();
	496	}
	497
	498	__set_current_state(TASK_RUNNING);
	499
	500	while (local_softirq_pending()) {
	501	/* Preempt disable stops cpu going offline.
	502	If already offline, we'll be on wrong CPU:
	503	don't process */
	504	if (cpu_is_offline((long)__bind_cpu))
	505	goto wait_to_die;
	506	do_softirq();
	507	preempt_enable_no_resched();
	508	cond_resched();
	509	preempt_disable();
	510	}
	511	preempt_enable();
	512	set_current_state(TASK_INTERRUPTIBLE);
	513	}
	514	__set_current_state(TASK_RUNNING);
	515	return 0;
	516
	517	wait_to_die:
	518	preempt_enable();
	519	/* Wait for kthread_stop */
	520	set_current_state(TASK_INTERRUPTIBLE);
	521	while (!kthread_should_stop()) {
	522	schedule();
	523	set_current_state(TASK_INTERRUPTIBLE);
	524	}
	525	__set_current_state(TASK_RUNNING);
	526	return 0;
	527	}
	528
	529	#ifdef CONFIG_HOTPLUG_CPU
	530	/*
	531	* tasklet_kill_immediate is called to remove a tasklet which can already be
	532	* scheduled for execution on @cpu.
	533	*
	534	* Unlike tasklet_kill, this function removes the tasklet
	535	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	536	*
	537	* When this function is called, @cpu must be in the CPU_DEAD state.
	538	*/
	539	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	540	{
	541	struct tasklet_struct **i;
	542
	543	BUG_ON(cpu_online(cpu));
	544	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	545
	546	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	547	return;
	548
	549	/* CPU is dead, so no lock needed. */
	550	for (i = &per_cpu(tasklet_vec, cpu).list; i; i = &(i)->next) {
	551	if (*i == t) {
552	*i = t->next;
553	return;
554	}
555	}
556	BUG();
557	}
558
559	static void takeover_tasklets(unsigned int cpu)
560	{
561	struct tasklet_struct **i;
562
563	/* CPU is dead, so no lock needed. */
564	local_irq_disable();
565
566	/* Find end, append list for that CPU. */
567	for (i = &__get_cpu_var(tasklet_vec).list; i; i = &(i)->next);
568	*i = per_cpu(tasklet_vec, cpu).list;
569	per_cpu(tasklet_vec, cpu).list = NULL;
570	raise_softirq_irqoff(TASKLET_SOFTIRQ);
571
572	for (i = &__get_cpu_var(tasklet_hi_vec).list; i; i = &(i)->next);
573	*i = per_cpu(tasklet_hi_vec, cpu).list;
574	per_cpu(tasklet_hi_vec, cpu).list = NULL;
575	raise_softirq_irqoff(HI_SOFTIRQ);
576
577	local_irq_enable();
578	}
579	#endif /* CONFIG_HOTPLUG_CPU */
580
8c78f307	581	static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4 LT	582	unsigned long action,
	583	void *hcpu)
	584	{
	585	int hotcpu = (unsigned long)hcpu;
	586	struct task_struct *p;
	587
	588	switch (action) {
	589	case CPU_UP_PREPARE:
8bb78442	590	case CPU_UP_PREPARE_FROZEN:
1da177e4 LT	591	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	592	if (IS_ERR(p)) {
	593	printk("ksoftirqd for %i failed\n", hotcpu);
	594	return NOTIFY_BAD;
	595	}
	596	kthread_bind(p, hotcpu);
	597	per_cpu(ksoftirqd, hotcpu) = p;
	598	break;
	599	case CPU_ONLINE:
8bb78442	600	case CPU_ONLINE_FROZEN:
1da177e4 LT	601	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	602	break;
	603	#ifdef CONFIG_HOTPLUG_CPU
	604	case CPU_UP_CANCELED:
8bb78442	605	case CPU_UP_CANCELED_FROZEN:
fc75cdfa HC	606	if (!per_cpu(ksoftirqd, hotcpu))
fc75cdfa HC	607	break;
1da177e4	608	/* Unbind so it can run. Fall thru. */
a4c4af7c HC	609	kthread_bind(per_cpu(ksoftirqd, hotcpu),
a4c4af7c HC	610	any_online_cpu(cpu_online_map));
1da177e4	611	case CPU_DEAD:
1c6b4aa9 ST	612	case CPU_DEAD_FROZEN: {
	613	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	614
1da177e4 LT	615	p = per_cpu(ksoftirqd, hotcpu);
1da177e4 LT	616	per_cpu(ksoftirqd, hotcpu) = NULL;
1c6b4aa9	617	sched_setscheduler(p, SCHED_FIFO, &param);
1da177e4 LT	618	kthread_stop(p);
	619	takeover_tasklets(hotcpu);
	620	break;
1c6b4aa9	621	}
1da177e4 LT	622	#endif /* CONFIG_HOTPLUG_CPU */
	623	}
	624	return NOTIFY_OK;
	625	}
	626
8c78f307	627	static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4 LT	628	.notifier_call = cpu_callback
	629	};
	630
	631	__init int spawn_ksoftirqd(void)
	632	{
	633	void cpu = (void )(long)smp_processor_id();
07dccf33 AM	634	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	635
	636	BUG_ON(err == NOTIFY_BAD);
1da177e4 LT	637	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	638	register_cpu_notifier(&cpu_nfb);
	639	return 0;
	640	}
78eef01b AM	641
	642	#ifdef CONFIG_SMP
	643	/*
	644	* Call a function on all processors
	645	*/
	646	int on_each_cpu(void (func) (void info), void *info, int retry, int wait)
	647	{
	648	int ret = 0;
	649
	650	preempt_disable();
	651	ret = smp_call_function(func, info, retry, wait);
	652	local_irq_disable();
	653	func(info);
	654	local_irq_enable();
	655	preempt_enable();
	656	return ret;
	657	}
	658	EXPORT_SYMBOL(on_each_cpu);
	659	#endif