[GitHub/mt8127/android_kernel_alcatel_ttab.git] / block / blk-ioc.c

/*
 * Functions related to io context handling
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/bootmem.h>	/* for max_pfn/max_low_pfn */
#include <linux/slab.h>

#include "blk.h"

/*
 * For io context allocations
 */
static struct kmem_cache *iocontext_cachep;

/**
 * get_io_context - increment reference count to io_context
 * @ioc: io_context to get
 *
 * Increment reference count to @ioc.
 */
void get_io_context(struct io_context *ioc)
{
	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
	atomic_long_inc(&ioc->refcount);
}
EXPORT_SYMBOL(get_io_context);

static void icq_free_icq_rcu(struct rcu_head *head)
{
	struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);

	kmem_cache_free(icq->__rcu_icq_cache, icq);
}

/*
 * Exit and free an icq.  Called with both ioc and q locked.
 */
static void ioc_exit_icq(struct io_cq *icq)
{
	struct io_context *ioc = icq->ioc;
	struct request_queue *q = icq->q;
	struct elevator_type *et = q->elevator->type;

	lockdep_assert_held(&ioc->lock);
	lockdep_assert_held(q->queue_lock);

	radix_tree_delete(&ioc->icq_tree, icq->q->id);
	hlist_del_init(&icq->ioc_node);
	list_del_init(&icq->q_node);

	/*
	 * Both setting lookup hint to and clearing it from @icq are done
	 * under queue_lock.  If it's not pointing to @icq now, it never
	 * will.  Hint assignment itself can race safely.
	 */
	if (rcu_dereference_raw(ioc->icq_hint) == icq)
		rcu_assign_pointer(ioc->icq_hint, NULL);

	if (et->ops.elevator_exit_icq_fn)
		et->ops.elevator_exit_icq_fn(icq);

	/*
	 * @icq->q might have gone away by the time RCU callback runs
	 * making it impossible to determine icq_cache.  Record it in @icq.
	 */
	icq->__rcu_icq_cache = et->icq_cache;
	call_rcu(&icq->__rcu_head, icq_free_icq_rcu);
}

/*
 * Slow path for ioc release in put_io_context().  Performs double-lock
 * dancing to unlink all icq's and then frees ioc.
 */
static void ioc_release_fn(struct work_struct *work)
{
	struct io_context *ioc = container_of(work, struct io_context,
					      release_work);
	struct request_queue *last_q = NULL;

	spin_lock_irq(&ioc->lock);

	while (!hlist_empty(&ioc->icq_list)) {
		struct io_cq *icq = hlist_entry(ioc->icq_list.first,
						struct io_cq, ioc_node);
		struct request_queue *this_q = icq->q;

		if (this_q != last_q) {
			/*
			 * Need to switch to @this_q.  Once we release
			 * @ioc->lock, it can go away along with @cic.
			 * Hold on to it.
			 */
			__blk_get_queue(this_q);

			/*
			 * blk_put_queue() might sleep thanks to kobject
			 * idiocy.  Always release both locks, put and
			 * restart.
			 */
			if (last_q) {
				spin_unlock(last_q->queue_lock);
				spin_unlock_irq(&ioc->lock);
				blk_put_queue(last_q);
			} else {
				spin_unlock_irq(&ioc->lock);
			}

			last_q = this_q;
			spin_lock_irq(this_q->queue_lock);
			spin_lock(&ioc->lock);
			continue;
		}
		ioc_exit_icq(icq);
	}

	if (last_q) {
		spin_unlock(last_q->queue_lock);
		spin_unlock_irq(&ioc->lock);
		blk_put_queue(last_q);
	} else {
		spin_unlock_irq(&ioc->lock);
	}

	kmem_cache_free(iocontext_cachep, ioc);
}

/**
 * put_io_context - put a reference of io_context
 * @ioc: io_context to put
 *
 * Decrement reference count of @ioc and release it if the count reaches
 * zero.
 */
void put_io_context(struct io_context *ioc)
{
	unsigned long flags;

	if (ioc == NULL)
		return;

	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);

	/*
	 * Releasing ioc requires reverse order double locking and we may
	 * already be holding a queue_lock.  Do it asynchronously from wq.
	 */
	if (atomic_long_dec_and_test(&ioc->refcount)) {
		spin_lock_irqsave(&ioc->lock, flags);
		if (!hlist_empty(&ioc->icq_list))
			schedule_work(&ioc->release_work);
		spin_unlock_irqrestore(&ioc->lock, flags);
	}
}
EXPORT_SYMBOL(put_io_context);

/* Called by the exiting task */
void exit_io_context(struct task_struct *task)
{
	struct io_context *ioc;

	task_lock(task);
	ioc = task->io_context;
	task->io_context = NULL;
	task_unlock(task);

	atomic_dec(&ioc->nr_tasks);
	put_io_context(ioc);
}

/**
 * ioc_clear_queue - break any ioc association with the specified queue
 * @q: request_queue being cleared
 *
 * Walk @q->icq_list and exit all io_cq's.  Must be called with @q locked.
 */
void ioc_clear_queue(struct request_queue *q)
{
	lockdep_assert_held(q->queue_lock);

	while (!list_empty(&q->icq_list)) {
		struct io_cq *icq = list_entry(q->icq_list.next,
					       struct io_cq, q_node);
		struct io_context *ioc = icq->ioc;

		spin_lock(&ioc->lock);
		ioc_exit_icq(icq);
		spin_unlock(&ioc->lock);
	}
}

void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,
				int node)
{
	struct io_context *ioc;

	ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
				    node);
	if (unlikely(!ioc))
		return;

	/* initialize */
	atomic_long_set(&ioc->refcount, 1);
	atomic_set(&ioc->nr_tasks, 1);
	spin_lock_init(&ioc->lock);
	INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);
	INIT_HLIST_HEAD(&ioc->icq_list);
	INIT_WORK(&ioc->release_work, ioc_release_fn);

	/*
	 * Try to install.  ioc shouldn't be installed if someone else
	 * already did or @task, which isn't %current, is exiting.  Note
	 * that we need to allow ioc creation on exiting %current as exit
	 * path may issue IOs from e.g. exit_files().  The exit path is
	 * responsible for not issuing IO after exit_io_context().
	 */
	task_lock(task);
	if (!task->io_context &&
	    (task == current || !(task->flags & PF_EXITING)))
		task->io_context = ioc;
	else
		kmem_cache_free(iocontext_cachep, ioc);
	task_unlock(task);
}

/**
 * get_task_io_context - get io_context of a task
 * @task: task of interest
 * @gfp_flags: allocation flags, used if allocation is necessary
 * @node: allocation node, used if allocation is necessary
 *
 * Return io_context of @task.  If it doesn't exist, it is created with
 * @gfp_flags and @node.  The returned io_context has its reference count
 * incremented.
 *
 * This function always goes through task_lock() and it's better to use
 * %current->io_context + get_io_context() for %current.
 */
struct io_context *get_task_io_context(struct task_struct *task,
				       gfp_t gfp_flags, int node)
{
	struct io_context *ioc;

	might_sleep_if(gfp_flags & __GFP_WAIT);

	do {
		task_lock(task);
		ioc = task->io_context;
		if (likely(ioc)) {
			get_io_context(ioc);
			task_unlock(task);
			return ioc;
		}
		task_unlock(task);
	} while (create_io_context(task, gfp_flags, node));

	return NULL;
}
EXPORT_SYMBOL(get_task_io_context);

/**
 * ioc_lookup_icq - lookup io_cq from ioc
 * @ioc: the associated io_context
 * @q: the associated request_queue
 *
 * Look up io_cq associated with @ioc - @q pair from @ioc.  Must be called
 * with @q->queue_lock held.
 */
struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q)
{
	struct io_cq *icq;

	lockdep_assert_held(q->queue_lock);

	/*
	 * icq's are indexed from @ioc using radix tree and hint pointer,
	 * both of which are protected with RCU.  All removals are done
	 * holding both q and ioc locks, and we're holding q lock - if we
	 * find a icq which points to us, it's guaranteed to be valid.
	 */
	rcu_read_lock();
	icq = rcu_dereference(ioc->icq_hint);
	if (icq && icq->q == q)
		goto out;

	icq = radix_tree_lookup(&ioc->icq_tree, q->id);
	if (icq && icq->q == q)
		rcu_assign_pointer(ioc->icq_hint, icq);	/* allowed to race */
	else
		icq = NULL;
out:
	rcu_read_unlock();
	return icq;
}
EXPORT_SYMBOL(ioc_lookup_icq);

/**
 * ioc_create_icq - create and link io_cq
 * @q: request_queue of interest
 * @gfp_mask: allocation mask
 *
 * Make sure io_cq linking %current->io_context and @q exists.  If either
 * io_context and/or icq don't exist, they will be created using @gfp_mask.
 *
 * The caller is responsible for ensuring @ioc won't go away and @q is
 * alive and will stay alive until this function returns.
 */
struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask)
{
	struct elevator_type *et = q->elevator->type;
	struct io_context *ioc;
	struct io_cq *icq;

	/* allocate stuff */
	ioc = create_io_context(current, gfp_mask, q->node);
	if (!ioc)
		return NULL;

	icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
				    q->node);
	if (!icq)
		return NULL;

	if (radix_tree_preload(gfp_mask) < 0) {
		kmem_cache_free(et->icq_cache, icq);
		return NULL;
	}

	icq->ioc = ioc;
	icq->q = q;
	INIT_LIST_HEAD(&icq->q_node);
	INIT_HLIST_NODE(&icq->ioc_node);

	/* lock both q and ioc and try to link @icq */
	spin_lock_irq(q->queue_lock);
	spin_lock(&ioc->lock);

	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
		hlist_add_head(&icq->ioc_node, &ioc->icq_list);
		list_add(&icq->q_node, &q->icq_list);
		if (et->ops.elevator_init_icq_fn)
			et->ops.elevator_init_icq_fn(icq);
	} else {
		kmem_cache_free(et->icq_cache, icq);
		icq = ioc_lookup_icq(ioc, q);
		if (!icq)
			printk(KERN_ERR "cfq: icq link failed!\n");
	}

	spin_unlock(&ioc->lock);
	spin_unlock_irq(q->queue_lock);
	radix_tree_preload_end();
	return icq;
}

void ioc_set_changed(struct io_context *ioc, int which)
{
	struct io_cq *icq;
	struct hlist_node *n;

	hlist_for_each_entry(icq, n, &ioc->icq_list, ioc_node)
		set_bit(which, &icq->changed);
}

/**
 * ioc_ioprio_changed - notify ioprio change
 * @ioc: io_context of interest
 * @ioprio: new ioprio
 *
 * @ioc's ioprio has changed to @ioprio.  Set %ICQ_IOPRIO_CHANGED for all
 * icq's.  iosched is responsible for checking the bit and applying it on
 * request issue path.
 */
void ioc_ioprio_changed(struct io_context *ioc, int ioprio)
{
	unsigned long flags;

	spin_lock_irqsave(&ioc->lock, flags);
	ioc->ioprio = ioprio;
	ioc_set_changed(ioc, ICQ_IOPRIO_CHANGED);
	spin_unlock_irqrestore(&ioc->lock, flags);
}

/**
 * ioc_cgroup_changed - notify cgroup change
 * @ioc: io_context of interest
 *
 * @ioc's cgroup has changed.  Set %ICQ_CGROUP_CHANGED for all icq's.
 * iosched is responsible for checking the bit and applying it on request
 * issue path.
 */
void ioc_cgroup_changed(struct io_context *ioc)
{
	unsigned long flags;

	spin_lock_irqsave(&ioc->lock, flags);
	ioc_set_changed(ioc, ICQ_CGROUP_CHANGED);
	spin_unlock_irqrestore(&ioc->lock, flags);
}
EXPORT_SYMBOL(ioc_cgroup_changed);

static int __init blk_ioc_init(void)
{
	iocontext_cachep = kmem_cache_create("blkdev_ioc",
			sizeof(struct io_context), 0, SLAB_PANIC, NULL);
	return 0;
}
subsys_initcall(blk_ioc_init);
Commit	Line	Data
86db1e29 JA	1	/*
	2	* Functions related to io context handling
	3	*/
	4	#include <linux/kernel.h>
	5	#include <linux/module.h>
	6	#include <linux/init.h>
	7	#include <linux/bio.h>
	8	#include <linux/blkdev.h>
	9	#include <linux/bootmem.h> /* for max_pfn/max_low_pfn */
5a0e3ad6	10	#include <linux/slab.h>
86db1e29 JA	11
	12	#include "blk.h"
	13
	14	/*
	15	* For io context allocations
	16	*/
	17	static struct kmem_cache *iocontext_cachep;
	18
6e736be7 TH	19	/**
	20	* get_io_context - increment reference count to io_context
	21	* @ioc: io_context to get
	22	*
	23	* Increment reference count to @ioc.
	24	*/
	25	void get_io_context(struct io_context *ioc)
	26	{
	27	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
	28	atomic_long_inc(&ioc->refcount);
	29	}
	30	EXPORT_SYMBOL(get_io_context);
	31
7e5a8794 TH	32	static void icq_free_icq_rcu(struct rcu_head *head)
	33	{
	34	struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);
	35
	36	kmem_cache_free(icq->__rcu_icq_cache, icq);
	37	}
	38
	39	/*
	40	* Exit and free an icq. Called with both ioc and q locked.
	41	*/
	42	static void ioc_exit_icq(struct io_cq *icq)
	43	{
	44	struct io_context *ioc = icq->ioc;
	45	struct request_queue *q = icq->q;
	46	struct elevator_type *et = q->elevator->type;
	47
	48	lockdep_assert_held(&ioc->lock);
	49	lockdep_assert_held(q->queue_lock);
	50
	51	radix_tree_delete(&ioc->icq_tree, icq->q->id);
	52	hlist_del_init(&icq->ioc_node);
	53	list_del_init(&icq->q_node);
	54
	55	/*
	56	* Both setting lookup hint to and clearing it from @icq are done
	57	* under queue_lock. If it's not pointing to @icq now, it never
	58	* will. Hint assignment itself can race safely.
	59	*/
	60	if (rcu_dereference_raw(ioc->icq_hint) == icq)
	61	rcu_assign_pointer(ioc->icq_hint, NULL);
	62
11a3122f	63	if (et->ops.elevator_exit_icq_fn)
7e5a8794	64	et->ops.elevator_exit_icq_fn(icq);
7e5a8794 TH	65
	66	/*
	67	* @icq->q might have gone away by the time RCU callback runs
	68	* making it impossible to determine icq_cache. Record it in @icq.
	69	*/
	70	icq->__rcu_icq_cache = et->icq_cache;
	71	call_rcu(&icq->__rcu_head, icq_free_icq_rcu);
	72	}
	73
b2efa052 TH	74	/*
b2efa052 TH	75	* Slow path for ioc release in put_io_context(). Performs double-lock
c5869807	76	* dancing to unlink all icq's and then frees ioc.
b2efa052 TH	77	*/
b2efa052 TH	78	static void ioc_release_fn(struct work_struct *work)
86db1e29	79	{
b2efa052 TH	80	struct io_context *ioc = container_of(work, struct io_context,
	81	release_work);
	82	struct request_queue *last_q = NULL;
	83
	84	spin_lock_irq(&ioc->lock);
	85
c5869807 TH	86	while (!hlist_empty(&ioc->icq_list)) {
	87	struct io_cq *icq = hlist_entry(ioc->icq_list.first,
	88	struct io_cq, ioc_node);
	89	struct request_queue *this_q = icq->q;
b2efa052 TH	90
	91	if (this_q != last_q) {
	92	/*
	93	* Need to switch to @this_q. Once we release
	94	* @ioc->lock, it can go away along with @cic.
	95	* Hold on to it.
	96	*/
	97	__blk_get_queue(this_q);
	98
	99	/*
	100	* blk_put_queue() might sleep thanks to kobject
	101	* idiocy. Always release both locks, put and
	102	* restart.
	103	*/
	104	if (last_q) {
	105	spin_unlock(last_q->queue_lock);
	106	spin_unlock_irq(&ioc->lock);
	107	blk_put_queue(last_q);
	108	} else {
	109	spin_unlock_irq(&ioc->lock);
	110	}
	111
	112	last_q = this_q;
	113	spin_lock_irq(this_q->queue_lock);
	114	spin_lock(&ioc->lock);
	115	continue;
	116	}
7e5a8794	117	ioc_exit_icq(icq);
b2efa052	118	}
ffc4e759	119
b2efa052 TH	120	if (last_q) {
	121	spin_unlock(last_q->queue_lock);
	122	spin_unlock_irq(&ioc->lock);
	123	blk_put_queue(last_q);
	124	} else {
	125	spin_unlock_irq(&ioc->lock);
ffc4e759	126	}
b2efa052 TH	127
b2efa052 TH	128	kmem_cache_free(iocontext_cachep, ioc);
86db1e29 JA	129	}
86db1e29 JA	130
42ec57a8 TH	131	/**
	132	* put_io_context - put a reference of io_context
	133	* @ioc: io_context to put
	134	*
	135	* Decrement reference count of @ioc and release it if the count reaches
11a3122f	136	* zero.
86db1e29	137	*/
11a3122f	138	void put_io_context(struct io_context *ioc)
86db1e29	139	{
b2efa052 TH	140	unsigned long flags;
b2efa052 TH	141
86db1e29	142	if (ioc == NULL)
42ec57a8	143	return;
86db1e29	144
42ec57a8	145	BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
86db1e29	146
b2efa052	147	/*
11a3122f TH	148	* Releasing ioc requires reverse order double locking and we may
11a3122f TH	149	* already be holding a queue_lock. Do it asynchronously from wq.
b2efa052	150	*/
11a3122f TH	151	if (atomic_long_dec_and_test(&ioc->refcount)) {
	152	spin_lock_irqsave(&ioc->lock, flags);
	153	if (!hlist_empty(&ioc->icq_list))
	154	schedule_work(&ioc->release_work);
	155	spin_unlock_irqrestore(&ioc->lock, flags);
b2efa052	156	}
86db1e29	157	}
b2efa052	158	EXPORT_SYMBOL(put_io_context);
86db1e29	159
27667c99	160	/* Called by the exiting task */
b69f2292	161	void exit_io_context(struct task_struct *task)
86db1e29 JA	162	{
	163	struct io_context *ioc;
	164
b69f2292 LR	165	task_lock(task);
	166	ioc = task->io_context;
	167	task->io_context = NULL;
	168	task_unlock(task);
86db1e29	169
b2efa052	170	atomic_dec(&ioc->nr_tasks);
11a3122f	171	put_io_context(ioc);
86db1e29 JA	172	}
86db1e29 JA	173
7e5a8794 TH	174	/**
	175	* ioc_clear_queue - break any ioc association with the specified queue
	176	* @q: request_queue being cleared
	177	*
	178	* Walk @q->icq_list and exit all io_cq's. Must be called with @q locked.
	179	*/
	180	void ioc_clear_queue(struct request_queue *q)
	181	{
	182	lockdep_assert_held(q->queue_lock);
	183
	184	while (!list_empty(&q->icq_list)) {
	185	struct io_cq *icq = list_entry(q->icq_list.next,
	186	struct io_cq, q_node);
	187	struct io_context *ioc = icq->ioc;
	188
	189	spin_lock(&ioc->lock);
	190	ioc_exit_icq(icq);
	191	spin_unlock(&ioc->lock);
	192	}
	193	}
	194
f2dbd76a TH	195	void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,
f2dbd76a TH	196	int node)
86db1e29	197	{
df415656	198	struct io_context *ioc;
86db1e29	199
42ec57a8 TH	200	ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags \| __GFP_ZERO,
	201	node);
	202	if (unlikely(!ioc))
f2dbd76a	203	return;
42ec57a8 TH	204
	205	/* initialize */
	206	atomic_long_set(&ioc->refcount, 1);
	207	atomic_set(&ioc->nr_tasks, 1);
	208	spin_lock_init(&ioc->lock);
c5869807 TH	209	INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC \| __GFP_HIGH);
c5869807 TH	210	INIT_HLIST_HEAD(&ioc->icq_list);
b2efa052	211	INIT_WORK(&ioc->release_work, ioc_release_fn);
86db1e29	212
fd638368 TH	213	/*
	214	* Try to install. ioc shouldn't be installed if someone else
	215	* already did or @task, which isn't %current, is exiting. Note
	216	* that we need to allow ioc creation on exiting %current as exit
	217	* path may issue IOs from e.g. exit_files(). The exit path is
	218	* responsible for not issuing IO after exit_io_context().
	219	*/
6e736be7	220	task_lock(task);
fd638368 TH	221	if (!task->io_context &&
fd638368 TH	222	(task == current \|\| !(task->flags & PF_EXITING)))
6e736be7	223	task->io_context = ioc;
f2dbd76a	224	else
6e736be7	225	kmem_cache_free(iocontext_cachep, ioc);
6e736be7	226	task_unlock(task);
86db1e29	227	}
86db1e29	228
6e736be7 TH	229	/**
	230	* get_task_io_context - get io_context of a task
	231	* @task: task of interest
	232	* @gfp_flags: allocation flags, used if allocation is necessary
	233	* @node: allocation node, used if allocation is necessary
	234	*
	235	* Return io_context of @task. If it doesn't exist, it is created with
	236	* @gfp_flags and @node. The returned io_context has its reference count
	237	* incremented.
86db1e29	238	*
6e736be7	239	* This function always goes through task_lock() and it's better to use
f2dbd76a	240	* %current->io_context + get_io_context() for %current.
86db1e29	241	*/
6e736be7 TH	242	struct io_context get_task_io_context(struct task_struct task,
6e736be7 TH	243	gfp_t gfp_flags, int node)
86db1e29	244	{
6e736be7	245	struct io_context *ioc;
86db1e29	246
6e736be7 TH	247	might_sleep_if(gfp_flags & __GFP_WAIT);
6e736be7 TH	248
f2dbd76a TH	249	do {
	250	task_lock(task);
	251	ioc = task->io_context;
	252	if (likely(ioc)) {
	253	get_io_context(ioc);
	254	task_unlock(task);
	255	return ioc;
	256	}
6e736be7	257	task_unlock(task);
f2dbd76a	258	} while (create_io_context(task, gfp_flags, node));
6e736be7	259
f2dbd76a	260	return NULL;
86db1e29	261	}
6e736be7	262	EXPORT_SYMBOL(get_task_io_context);
86db1e29	263
47fdd4ca TH	264	/**
	265	* ioc_lookup_icq - lookup io_cq from ioc
	266	* @ioc: the associated io_context
	267	* @q: the associated request_queue
	268	*
	269	* Look up io_cq associated with @ioc - @q pair from @ioc. Must be called
	270	* with @q->queue_lock held.
	271	*/
	272	struct io_cq ioc_lookup_icq(struct io_context ioc, struct request_queue *q)
	273	{
	274	struct io_cq *icq;
	275
	276	lockdep_assert_held(q->queue_lock);
	277
	278	/*
	279	* icq's are indexed from @ioc using radix tree and hint pointer,
	280	* both of which are protected with RCU. All removals are done
	281	* holding both q and ioc locks, and we're holding q lock - if we
	282	* find a icq which points to us, it's guaranteed to be valid.
	283	*/
	284	rcu_read_lock();
	285	icq = rcu_dereference(ioc->icq_hint);
	286	if (icq && icq->q == q)
	287	goto out;
	288
	289	icq = radix_tree_lookup(&ioc->icq_tree, q->id);
	290	if (icq && icq->q == q)
	291	rcu_assign_pointer(ioc->icq_hint, icq); /* allowed to race */
	292	else
	293	icq = NULL;
	294	out:
	295	rcu_read_unlock();
	296	return icq;
	297	}
	298	EXPORT_SYMBOL(ioc_lookup_icq);
	299
f1f8cc94 TH	300	/**
	301	* ioc_create_icq - create and link io_cq
	302	* @q: request_queue of interest
	303	* @gfp_mask: allocation mask
	304	*
	305	* Make sure io_cq linking %current->io_context and @q exists. If either
	306	* io_context and/or icq don't exist, they will be created using @gfp_mask.
	307	*
	308	* The caller is responsible for ensuring @ioc won't go away and @q is
	309	* alive and will stay alive until this function returns.
	310	*/
	311	struct io_cq ioc_create_icq(struct request_queue q, gfp_t gfp_mask)
	312	{
	313	struct elevator_type *et = q->elevator->type;
	314	struct io_context *ioc;
	315	struct io_cq *icq;
	316
	317	/* allocate stuff */
	318	ioc = create_io_context(current, gfp_mask, q->node);
	319	if (!ioc)
	320	return NULL;
	321
	322	icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask \| __GFP_ZERO,
	323	q->node);
	324	if (!icq)
	325	return NULL;
	326
	327	if (radix_tree_preload(gfp_mask) < 0) {
	328	kmem_cache_free(et->icq_cache, icq);
	329	return NULL;
	330	}
	331
	332	icq->ioc = ioc;
	333	icq->q = q;
	334	INIT_LIST_HEAD(&icq->q_node);
	335	INIT_HLIST_NODE(&icq->ioc_node);
	336
	337	/* lock both q and ioc and try to link @icq */
	338	spin_lock_irq(q->queue_lock);
	339	spin_lock(&ioc->lock);
	340
	341	if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
	342	hlist_add_head(&icq->ioc_node, &ioc->icq_list);
	343	list_add(&icq->q_node, &q->icq_list);
	344	if (et->ops.elevator_init_icq_fn)
	345	et->ops.elevator_init_icq_fn(icq);
	346	} else {
	347	kmem_cache_free(et->icq_cache, icq);
	348	icq = ioc_lookup_icq(ioc, q);
	349	if (!icq)
	350	printk(KERN_ERR "cfq: icq link failed!\n");
	351	}
	352
	353	spin_unlock(&ioc->lock);
	354	spin_unlock_irq(q->queue_lock);
	355	radix_tree_preload_end();
	356	return icq;
	357	}
	358
dc86900e TH	359	void ioc_set_changed(struct io_context *ioc, int which)
dc86900e TH	360	{
c5869807	361	struct io_cq *icq;
dc86900e TH	362	struct hlist_node *n;
dc86900e TH	363
c5869807 TH	364	hlist_for_each_entry(icq, n, &ioc->icq_list, ioc_node)
c5869807 TH	365	set_bit(which, &icq->changed);
dc86900e TH	366	}
	367
	368	/**
	369	* ioc_ioprio_changed - notify ioprio change
	370	* @ioc: io_context of interest
	371	* @ioprio: new ioprio
	372	*
c5869807 TH	373	* @ioc's ioprio has changed to @ioprio. Set %ICQ_IOPRIO_CHANGED for all
c5869807 TH	374	* icq's. iosched is responsible for checking the bit and applying it on
dc86900e TH	375	* request issue path.
	376	*/
	377	void ioc_ioprio_changed(struct io_context *ioc, int ioprio)
	378	{
	379	unsigned long flags;
	380
	381	spin_lock_irqsave(&ioc->lock, flags);
	382	ioc->ioprio = ioprio;
c5869807	383	ioc_set_changed(ioc, ICQ_IOPRIO_CHANGED);
dc86900e TH	384	spin_unlock_irqrestore(&ioc->lock, flags);
	385	}
	386
	387	/**
	388	* ioc_cgroup_changed - notify cgroup change
	389	* @ioc: io_context of interest
	390	*
c5869807	391	* @ioc's cgroup has changed. Set %ICQ_CGROUP_CHANGED for all icq's.
dc86900e TH	392	* iosched is responsible for checking the bit and applying it on request
	393	* issue path.
	394	*/
	395	void ioc_cgroup_changed(struct io_context *ioc)
	396	{
	397	unsigned long flags;
	398
	399	spin_lock_irqsave(&ioc->lock, flags);
c5869807	400	ioc_set_changed(ioc, ICQ_CGROUP_CHANGED);
dc86900e TH	401	spin_unlock_irqrestore(&ioc->lock, flags);
dc86900e TH	402	}
64c42998	403	EXPORT_SYMBOL(ioc_cgroup_changed);
dc86900e	404
13341598	405	static int __init blk_ioc_init(void)
86db1e29 JA	406	{
	407	iocontext_cachep = kmem_cache_create("blkdev_ioc",
	408	sizeof(struct io_context), 0, SLAB_PANIC, NULL);
	409	return 0;
	410	}
	411	subsys_initcall(blk_ioc_init);