[GitHub/mt8127/android_kernel_alcatel_ttab.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 *	Distribute under GPLv2.
 *
 *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

static inline void _local_bh_enable_ip(unsigned long ip)
{
	WARN_ON_ONCE(in_irq() || irqs_disabled());
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_disable();
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_enable();
#endif
	preempt_check_resched();
}

void local_bh_enable(void)
{
	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
	_local_bh_enable_ip(ip);
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			int prev_count = preempt_count();

			h->action(h);

			if (unlikely(prev_count != preempt_count())) {
				printk(KERN_ERR "huh, entered softirq %td %p"
				       "with preempt_count %08x,"
				       " exited with %08x?\n", h - softirq_vec,
				       h->action, prev_count, preempt_count());
				preempt_count() = prev_count;
			}

			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
#ifdef CONFIG_NO_HZ
	int cpu = smp_processor_id();
	if (idle_cpu(cpu) && !in_interrupt())
		tick_nohz_stop_idle(cpu);
#endif
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(cpu))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick(0);
	rcu_irq_exit();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

void raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action *))
{
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *head;
	struct tasklet_struct **tail;
};

static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);

void __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_vec).tail = t;
	__get_cpu_var(tasklet_vec).tail = &(t->next);
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = NULL;
	*__get_cpu_var(tasklet_hi_vec).tail = t;
	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).head;
	__get_cpu_var(tasklet_vec).head = NULL;
	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_vec).tail = t;
		__get_cpu_var(tasklet_vec).tail = &(t->next);
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).head;
	__get_cpu_var(tasklet_hi_vec).head = NULL;
	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = NULL;
		*__get_cpu_var(tasklet_hi_vec).tail = t;
		__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	int cpu;

	for_each_possible_cpu(cpu) {
		per_cpu(tasklet_vec, cpu).tail =
			&per_cpu(tasklet_vec, cpu).head;
		per_cpu(tasklet_hi_vec, cpu).tail =
			&per_cpu(tasklet_hi_vec, cpu).head;
	}

	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
}

static int ksoftirqd(void * __bind_cpu)
{
	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			/* If this was the tail element, move the tail ptr */
			if (*i == NULL)
				per_cpu(tasklet_vec, cpu).tail = i;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
		*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
		__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
		per_cpu(tasklet_vec, cpu).head = NULL;
		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	}
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
		*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
		__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
		per_cpu(tasklet_hi_vec, cpu).head = NULL;
		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	}
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

static __init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}
early_initcall(spawn_ksoftirqd);

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
b10db7f0 PM	6	* Distribute under GPLv2.
	7	*
	8	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
1da177e4 LT	9	*/
	10
	11	#include <linux/module.h>
	12	#include <linux/kernel_stat.h>
	13	#include <linux/interrupt.h>
	14	#include <linux/init.h>
	15	#include <linux/mm.h>
	16	#include <linux/notifier.h>
	17	#include <linux/percpu.h>
	18	#include <linux/cpu.h>
83144186	19	#include <linux/freezer.h>
1da177e4 LT	20	#include <linux/kthread.h>
1da177e4 LT	21	#include <linux/rcupdate.h>
78eef01b	22	#include <linux/smp.h>
79bf2bb3	23	#include <linux/tick.h>
1da177e4 LT	24
	25	#include <asm/irq.h>
	26	/*
	27	- No shared variables, all the data are CPU local.
	28	- If a softirq needs serialization, let it serialize itself
	29	by its own spinlocks.
	30	- Even if softirq is serialized, only local cpu is marked for
	31	execution. Hence, we get something sort of weak cpu binding.
	32	Though it is still not clear, will it result in better locality
	33	or will not.
	34
	35	Examples:
	36	- NET RX softirq. It is multithreaded and does not require
	37	any global serialization.
	38	- NET TX softirq. It kicks software netdevice queues, hence
	39	it is logically serialized per device, but this serialization
	40	is invisible to common code.
	41	- Tasklets: serialized wrt itself.
	42	*/
	43
	44	#ifndef __ARCH_IRQ_STAT
	45	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	46	EXPORT_SYMBOL(irq_stat);
	47	#endif
	48
978b0116	49	static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
1da177e4 LT	50
	51	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	52
	53	/*
	54	* we cannot loop indefinitely here to avoid userspace starvation,
	55	* but we also don't want to introduce a worst case 1/HZ latency
	56	* to the pending events, so lets the scheduler to balance
	57	* the softirq load for us.
	58	*/
	59	static inline void wakeup_softirqd(void)
	60	{
	61	/* Interrupts are disabled: no need to stop preemption */
	62	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	63
	64	if (tsk && tsk->state != TASK_RUNNING)
	65	wake_up_process(tsk);
	66	}
	67
de30a2b3 IM	68	/*
	69	* This one is for softirq.c-internal use,
	70	* where hardirqs are disabled legitimately:
	71	*/
3c829c36	72	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	73	static void __local_bh_disable(unsigned long ip)
	74	{
	75	unsigned long flags;
	76
	77	WARN_ON_ONCE(in_irq());
	78
	79	raw_local_irq_save(flags);
	80	add_preempt_count(SOFTIRQ_OFFSET);
	81	/*
	82	* Were softirqs turned off above:
	83	*/
	84	if (softirq_count() == SOFTIRQ_OFFSET)
	85	trace_softirqs_off(ip);
	86	raw_local_irq_restore(flags);
	87	}
3c829c36 TC	88	#else /* !CONFIG_TRACE_IRQFLAGS */
	89	static inline void __local_bh_disable(unsigned long ip)
	90	{
	91	add_preempt_count(SOFTIRQ_OFFSET);
	92	barrier();
	93	}
	94	#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 IM	95
	96	void local_bh_disable(void)
	97	{
	98	__local_bh_disable((unsigned long)__builtin_return_address(0));
	99	}
	100
	101	EXPORT_SYMBOL(local_bh_disable);
	102
	103	void __local_bh_enable(void)
	104	{
	105	WARN_ON_ONCE(in_irq());
	106
	107	/*
	108	* softirqs should never be enabled by __local_bh_enable(),
	109	* it always nests inside local_bh_enable() sections:
	110	*/
	111	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	112
	113	sub_preempt_count(SOFTIRQ_OFFSET);
	114	}
	115	EXPORT_SYMBOL_GPL(__local_bh_enable);
	116
	117	/*
	118	* Special-case - softirqs can safely be enabled in
	119	* cond_resched_softirq(), or by __do_softirq(),
	120	* without processing still-pending softirqs:
	121	*/
	122	void _local_bh_enable(void)
	123	{
	124	WARN_ON_ONCE(in_irq());
	125	WARN_ON_ONCE(!irqs_disabled());
	126
	127	if (softirq_count() == SOFTIRQ_OFFSET)
	128	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	129	sub_preempt_count(SOFTIRQ_OFFSET);
	130	}
	131
	132	EXPORT_SYMBOL(_local_bh_enable);
	133
0f476b6d	134	static inline void _local_bh_enable_ip(unsigned long ip)
de30a2b3	135	{
0f476b6d	136	WARN_ON_ONCE(in_irq() \|\| irqs_disabled());
3c829c36	137	#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d	138	local_irq_disable();
3c829c36	139	#endif
de30a2b3 IM	140	/*
	141	* Are softirqs going to be turned on now:
	142	*/
	143	if (softirq_count() == SOFTIRQ_OFFSET)
0f476b6d	144	trace_softirqs_on(ip);
de30a2b3 IM	145	/*
	146	* Keep preemption disabled until we are done with
	147	* softirq processing:
	148	*/
	149	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	150
	151	if (unlikely(!in_interrupt() && local_softirq_pending()))
	152	do_softirq();
	153
	154	dec_preempt_count();
3c829c36	155	#ifdef CONFIG_TRACE_IRQFLAGS
0f476b6d	156	local_irq_enable();
3c829c36	157	#endif
de30a2b3 IM	158	preempt_check_resched();
de30a2b3 IM	159	}
0f476b6d JB	160
	161	void local_bh_enable(void)
	162	{
	163	_local_bh_enable_ip((unsigned long)__builtin_return_address(0));
	164	}
de30a2b3 IM	165	EXPORT_SYMBOL(local_bh_enable);
	166
	167	void local_bh_enable_ip(unsigned long ip)
	168	{
0f476b6d	169	_local_bh_enable_ip(ip);
de30a2b3 IM	170	}
	171	EXPORT_SYMBOL(local_bh_enable_ip);
	172
1da177e4 LT	173	/*
	174	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	175	* and we fall back to softirqd after that.
	176	*
	177	* This number has been established via experimentation.
	178	* The two things to balance is latency against fairness -
	179	* we want to handle softirqs as soon as possible, but they
	180	* should not be able to lock up the box.
	181	*/
	182	#define MAX_SOFTIRQ_RESTART 10
	183
	184	asmlinkage void __do_softirq(void)
	185	{
	186	struct softirq_action *h;
	187	__u32 pending;
	188	int max_restart = MAX_SOFTIRQ_RESTART;
	189	int cpu;
	190
	191	pending = local_softirq_pending();
829035fd PM	192	account_system_vtime(current);
829035fd PM	193
de30a2b3 IM	194	__local_bh_disable((unsigned long)__builtin_return_address(0));
de30a2b3 IM	195	trace_softirq_enter();
1da177e4	196
1da177e4 LT	197	cpu = smp_processor_id();
	198	restart:
	199	/* Reset the pending bitmask before enabling irqs */
3f74478b	200	set_softirq_pending(0);
1da177e4	201
c70f5d66	202	local_irq_enable();
1da177e4 LT	203
	204	h = softirq_vec;
	205
	206	do {
	207	if (pending & 1) {
8e85b4b5 TG	208	int prev_count = preempt_count();
8e85b4b5 TG	209
1da177e4	210	h->action(h);
8e85b4b5 TG	211
8e85b4b5 TG	212	if (unlikely(prev_count != preempt_count())) {
1c95e1b6	213	printk(KERN_ERR "huh, entered softirq %td %p"
8e85b4b5 TG	214	"with preempt_count %08x,"
	215	" exited with %08x?\n", h - softirq_vec,
	216	h->action, prev_count, preempt_count());
	217	preempt_count() = prev_count;
	218	}
	219
1da177e4 LT	220	rcu_bh_qsctr_inc(cpu);
	221	}
	222	h++;
	223	pending >>= 1;
	224	} while (pending);
	225
c70f5d66	226	local_irq_disable();
1da177e4 LT	227
	228	pending = local_softirq_pending();
	229	if (pending && --max_restart)
	230	goto restart;
	231
	232	if (pending)
	233	wakeup_softirqd();
	234
de30a2b3	235	trace_softirq_exit();
829035fd PM	236
829035fd PM	237	account_system_vtime(current);
de30a2b3	238	_local_bh_enable();
1da177e4 LT	239	}
	240
	241	#ifndef __ARCH_HAS_DO_SOFTIRQ
	242
	243	asmlinkage void do_softirq(void)
	244	{
	245	__u32 pending;
	246	unsigned long flags;
	247
	248	if (in_interrupt())
	249	return;
	250
	251	local_irq_save(flags);
	252
	253	pending = local_softirq_pending();
	254
	255	if (pending)
	256	__do_softirq();
	257
	258	local_irq_restore(flags);
	259	}
	260
1da177e4 LT	261	#endif
1da177e4 LT	262
dde4b2b5 IM	263	/*
	264	* Enter an interrupt context.
	265	*/
	266	void irq_enter(void)
	267	{
6378ddb5 VP	268	#ifdef CONFIG_NO_HZ
	269	int cpu = smp_processor_id();
	270	if (idle_cpu(cpu) && !in_interrupt())
	271	tick_nohz_stop_idle(cpu);
	272	#endif
79bf2bb3 TG	273	__irq_enter();
79bf2bb3 TG	274	#ifdef CONFIG_NO_HZ
6378ddb5	275	if (idle_cpu(cpu))
79bf2bb3 TG	276	tick_nohz_update_jiffies();
79bf2bb3 TG	277	#endif
dde4b2b5 IM	278	}
dde4b2b5 IM	279
1da177e4 LT	280	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	281	# define invoke_softirq() __do_softirq()
	282	#else
	283	# define invoke_softirq() do_softirq()
	284	#endif
	285
	286	/*
	287	* Exit an interrupt context. Process softirqs if needed and possible:
	288	*/
	289	void irq_exit(void)
	290	{
	291	account_system_vtime(current);
de30a2b3	292	trace_hardirq_exit();
1da177e4 LT	293	sub_preempt_count(IRQ_EXIT_OFFSET);
	294	if (!in_interrupt() && local_softirq_pending())
	295	invoke_softirq();
79bf2bb3 TG	296
	297	#ifdef CONFIG_NO_HZ
	298	/* Make sure that timer wheel updates are propagated */
	299	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
b8f8c3cf	300	tick_nohz_stop_sched_tick(0);
2232c2d8	301	rcu_irq_exit();
79bf2bb3	302	#endif
1da177e4 LT	303	preempt_enable_no_resched();
	304	}
	305
	306	/*
	307	* This function must run with irqs disabled!
	308	*/
7ad5b3a5	309	inline void raise_softirq_irqoff(unsigned int nr)
1da177e4 LT	310	{
	311	__raise_softirq_irqoff(nr);
	312
	313	/*
	314	* If we're in an interrupt or softirq, we're done
	315	* (this also catches softirq-disabled code). We will
	316	* actually run the softirq once we return from
	317	* the irq or softirq.
	318	*
	319	* Otherwise we wake up ksoftirqd to make sure we
	320	* schedule the softirq soon.
	321	*/
	322	if (!in_interrupt())
	323	wakeup_softirqd();
	324	}
	325
7ad5b3a5	326	void raise_softirq(unsigned int nr)
1da177e4 LT	327	{
	328	unsigned long flags;
	329
	330	local_irq_save(flags);
	331	raise_softirq_irqoff(nr);
	332	local_irq_restore(flags);
	333	}
	334
962cf36c	335	void open_softirq(int nr, void (action)(struct softirq_action ))
1da177e4	336	{
1da177e4 LT	337	softirq_vec[nr].action = action;
	338	}
	339
1da177e4 LT	340	/* Tasklets */
	341	struct tasklet_head
	342	{
48f20a9a OJ	343	struct tasklet_struct *head;
48f20a9a OJ	344	struct tasklet_struct **tail;
1da177e4 LT	345	};
1da177e4 LT	346
4620b49f VN	347	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
4620b49f VN	348	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
1da177e4	349
7ad5b3a5	350	void __tasklet_schedule(struct tasklet_struct *t)
1da177e4 LT	351	{
	352	unsigned long flags;
	353
	354	local_irq_save(flags);
48f20a9a OJ	355	t->next = NULL;
	356	*__get_cpu_var(tasklet_vec).tail = t;
	357	__get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4 LT	358	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	359	local_irq_restore(flags);
	360	}
	361
	362	EXPORT_SYMBOL(__tasklet_schedule);
	363
7ad5b3a5	364	void __tasklet_hi_schedule(struct tasklet_struct *t)
1da177e4 LT	365	{
	366	unsigned long flags;
	367
	368	local_irq_save(flags);
48f20a9a OJ	369	t->next = NULL;
	370	*__get_cpu_var(tasklet_hi_vec).tail = t;
	371	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4 LT	372	raise_softirq_irqoff(HI_SOFTIRQ);
	373	local_irq_restore(flags);
	374	}
	375
	376	EXPORT_SYMBOL(__tasklet_hi_schedule);
	377
	378	static void tasklet_action(struct softirq_action *a)
	379	{
	380	struct tasklet_struct *list;
	381
	382	local_irq_disable();
48f20a9a OJ	383	list = __get_cpu_var(tasklet_vec).head;
	384	__get_cpu_var(tasklet_vec).head = NULL;
	385	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
1da177e4 LT	386	local_irq_enable();
	387
	388	while (list) {
	389	struct tasklet_struct *t = list;
	390
	391	list = list->next;
	392
	393	if (tasklet_trylock(t)) {
	394	if (!atomic_read(&t->count)) {
	395	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	396	BUG();
	397	t->func(t->data);
	398	tasklet_unlock(t);
	399	continue;
	400	}
	401	tasklet_unlock(t);
	402	}
	403
	404	local_irq_disable();
48f20a9a OJ	405	t->next = NULL;
	406	*__get_cpu_var(tasklet_vec).tail = t;
	407	__get_cpu_var(tasklet_vec).tail = &(t->next);
1da177e4 LT	408	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
	409	local_irq_enable();
	410	}
	411	}
	412
	413	static void tasklet_hi_action(struct softirq_action *a)
	414	{
	415	struct tasklet_struct *list;
	416
	417	local_irq_disable();
48f20a9a OJ	418	list = __get_cpu_var(tasklet_hi_vec).head;
	419	__get_cpu_var(tasklet_hi_vec).head = NULL;
	420	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
1da177e4 LT	421	local_irq_enable();
	422
	423	while (list) {
	424	struct tasklet_struct *t = list;
	425
	426	list = list->next;
	427
	428	if (tasklet_trylock(t)) {
	429	if (!atomic_read(&t->count)) {
	430	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	431	BUG();
	432	t->func(t->data);
	433	tasklet_unlock(t);
	434	continue;
	435	}
	436	tasklet_unlock(t);
	437	}
	438
	439	local_irq_disable();
48f20a9a OJ	440	t->next = NULL;
	441	*__get_cpu_var(tasklet_hi_vec).tail = t;
	442	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
1da177e4 LT	443	__raise_softirq_irqoff(HI_SOFTIRQ);
	444	local_irq_enable();
	445	}
	446	}
	447
	448
	449	void tasklet_init(struct tasklet_struct *t,
	450	void (*func)(unsigned long), unsigned long data)
	451	{
	452	t->next = NULL;
	453	t->state = 0;
	454	atomic_set(&t->count, 0);
	455	t->func = func;
	456	t->data = data;
	457	}
	458
	459	EXPORT_SYMBOL(tasklet_init);
	460
	461	void tasklet_kill(struct tasklet_struct *t)
	462	{
	463	if (in_interrupt())
	464	printk("Attempt to kill tasklet from interrupt\n");
	465
	466	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
	467	do
	468	yield();
	469	while (test_bit(TASKLET_STATE_SCHED, &t->state));
	470	}
	471	tasklet_unlock_wait(t);
	472	clear_bit(TASKLET_STATE_SCHED, &t->state);
	473	}
	474
	475	EXPORT_SYMBOL(tasklet_kill);
	476
	477	void __init softirq_init(void)
	478	{
48f20a9a OJ	479	int cpu;
	480
	481	for_each_possible_cpu(cpu) {
	482	per_cpu(tasklet_vec, cpu).tail =
	483	&per_cpu(tasklet_vec, cpu).head;
	484	per_cpu(tasklet_hi_vec, cpu).tail =
	485	&per_cpu(tasklet_hi_vec, cpu).head;
	486	}
	487
962cf36c CM	488	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
962cf36c CM	489	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
1da177e4 LT	490	}
	491
	492	static int ksoftirqd(void * __bind_cpu)
	493	{
1da177e4 LT	494	set_current_state(TASK_INTERRUPTIBLE);
	495
	496	while (!kthread_should_stop()) {
	497	preempt_disable();
	498	if (!local_softirq_pending()) {
	499	preempt_enable_no_resched();
	500	schedule();
	501	preempt_disable();
	502	}
	503
	504	__set_current_state(TASK_RUNNING);
	505
	506	while (local_softirq_pending()) {
	507	/* Preempt disable stops cpu going offline.
	508	If already offline, we'll be on wrong CPU:
	509	don't process */
	510	if (cpu_is_offline((long)__bind_cpu))
	511	goto wait_to_die;
	512	do_softirq();
	513	preempt_enable_no_resched();
	514	cond_resched();
	515	preempt_disable();
	516	}
	517	preempt_enable();
	518	set_current_state(TASK_INTERRUPTIBLE);
	519	}
	520	__set_current_state(TASK_RUNNING);
	521	return 0;
	522
	523	wait_to_die:
	524	preempt_enable();
	525	/* Wait for kthread_stop */
	526	set_current_state(TASK_INTERRUPTIBLE);
	527	while (!kthread_should_stop()) {
	528	schedule();
	529	set_current_state(TASK_INTERRUPTIBLE);
	530	}
	531	__set_current_state(TASK_RUNNING);
	532	return 0;
	533	}
	534
	535	#ifdef CONFIG_HOTPLUG_CPU
	536	/*
	537	* tasklet_kill_immediate is called to remove a tasklet which can already be
	538	* scheduled for execution on @cpu.
	539	*
	540	* Unlike tasklet_kill, this function removes the tasklet
	541	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	542	*
	543	* When this function is called, @cpu must be in the CPU_DEAD state.
	544	*/
	545	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	546	{
	547	struct tasklet_struct **i;
	548
	549	BUG_ON(cpu_online(cpu));
	550	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	551
	552	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	553	return;
	554
	555	/* CPU is dead, so no lock needed. */
48f20a9a	556	for (i = &per_cpu(tasklet_vec, cpu).head; i; i = &(i)->next) {
1da177e4 LT	557	if (*i == t) {
1da177e4 LT	558	*i = t->next;
48f20a9a OJ	559	/* If this was the tail element, move the tail ptr */
	560	if (*i == NULL)
	561	per_cpu(tasklet_vec, cpu).tail = i;
1da177e4 LT	562	return;
	563	}
	564	}
	565	BUG();
	566	}
	567
	568	static void takeover_tasklets(unsigned int cpu)
	569	{
1da177e4 LT	570	/* CPU is dead, so no lock needed. */
	571	local_irq_disable();
	572
	573	/* Find end, append list for that CPU. */
e5e41723 CB	574	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
	575	*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
	576	__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
	577	per_cpu(tasklet_vec, cpu).head = NULL;
	578	per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
	579	}
1da177e4 LT	580	raise_softirq_irqoff(TASKLET_SOFTIRQ);
1da177e4 LT	581
e5e41723 CB	582	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
	583	*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
	584	__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
	585	per_cpu(tasklet_hi_vec, cpu).head = NULL;
	586	per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
	587	}
1da177e4 LT	588	raise_softirq_irqoff(HI_SOFTIRQ);
	589
	590	local_irq_enable();
	591	}
	592	#endif /* CONFIG_HOTPLUG_CPU */
	593
8c78f307	594	static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4 LT	595	unsigned long action,
	596	void *hcpu)
	597	{
	598	int hotcpu = (unsigned long)hcpu;
	599	struct task_struct *p;
	600
	601	switch (action) {
	602	case CPU_UP_PREPARE:
8bb78442	603	case CPU_UP_PREPARE_FROZEN:
1da177e4 LT	604	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	605	if (IS_ERR(p)) {
	606	printk("ksoftirqd for %i failed\n", hotcpu);
	607	return NOTIFY_BAD;
	608	}
	609	kthread_bind(p, hotcpu);
	610	per_cpu(ksoftirqd, hotcpu) = p;
	611	break;
	612	case CPU_ONLINE:
8bb78442	613	case CPU_ONLINE_FROZEN:
1da177e4 LT	614	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	615	break;
	616	#ifdef CONFIG_HOTPLUG_CPU
	617	case CPU_UP_CANCELED:
8bb78442	618	case CPU_UP_CANCELED_FROZEN:
fc75cdfa HC	619	if (!per_cpu(ksoftirqd, hotcpu))
fc75cdfa HC	620	break;
1da177e4	621	/* Unbind so it can run. Fall thru. */
a4c4af7c HC	622	kthread_bind(per_cpu(ksoftirqd, hotcpu),
a4c4af7c HC	623	any_online_cpu(cpu_online_map));
1da177e4	624	case CPU_DEAD:
1c6b4aa9 ST	625	case CPU_DEAD_FROZEN: {
	626	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	627
1da177e4 LT	628	p = per_cpu(ksoftirqd, hotcpu);
1da177e4 LT	629	per_cpu(ksoftirqd, hotcpu) = NULL;
961ccddd	630	sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
1da177e4 LT	631	kthread_stop(p);
	632	takeover_tasklets(hotcpu);
	633	break;
1c6b4aa9	634	}
1da177e4 LT	635	#endif /* CONFIG_HOTPLUG_CPU */
	636	}
	637	return NOTIFY_OK;
	638	}
	639
8c78f307	640	static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4 LT	641	.notifier_call = cpu_callback
	642	};
	643
7babe8db	644	static __init int spawn_ksoftirqd(void)
1da177e4 LT	645	{
1da177e4 LT	646	void cpu = (void )(long)smp_processor_id();
07dccf33 AM	647	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	648
	649	BUG_ON(err == NOTIFY_BAD);
1da177e4 LT	650	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	651	register_cpu_notifier(&cpu_nfb);
	652	return 0;
	653	}
7babe8db	654	early_initcall(spawn_ksoftirqd);
78eef01b AM	655
	656	#ifdef CONFIG_SMP
	657	/*
	658	* Call a function on all processors
	659	*/
15c8b6c1	660	int on_each_cpu(void (func) (void info), void *info, int wait)
78eef01b AM	661	{
	662	int ret = 0;
	663
	664	preempt_disable();
8691e5a8	665	ret = smp_call_function(func, info, wait);
78eef01b AM	666	local_irq_disable();
	667	func(info);
	668	local_irq_enable();
	669	preempt_enable();
	670	return ret;
	671	}
	672	EXPORT_SYMBOL(on_each_cpu);
	673	#endif