[GitHub/LineageOS/G12/android_kernel_amlogic_linux-4.9.git] / block / blk-mq-tag.c

/*
 * Tag allocation using scalable bitmaps. Uses active queue tracking to support
 * fairer distribution of tags between multiple submitters when a shared tag map
 * is used.
 *
 * Copyright (C) 2013-2014 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
{
	if (!tags)
		return true;

	return sbitmap_any_bit_clear(&tags->bitmap_tags.sb);
}

/*
 * If a previously inactive queue goes active, bump the active user count.
 */
bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
{
	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
	    !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		atomic_inc(&hctx->tags->active_queues);

	return true;
}

/*
 * Wakeup all potentially sleeping on tags
 */
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
{
	sbitmap_queue_wake_all(&tags->bitmap_tags);
	if (include_reserve)
		sbitmap_queue_wake_all(&tags->breserved_tags);
}

/*
 * If a previously busy queue goes inactive, potential waiters could now
 * be allowed to queue. Wake them up and check.
 */
void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_tags *tags = hctx->tags;

	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		return;

	atomic_dec(&tags->active_queues);

	blk_mq_tag_wakeup_all(tags, false);
}

/*
 * For shared tag users, we track the number of currently active users
 * and attempt to provide a fair share of the tag depth for each of them.
 */
static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
				  struct sbitmap_queue *bt)
{
	unsigned int depth, users;

	if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED))
		return true;
	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
		return true;

	/*
	 * Don't try dividing an ant
	 */
	if (bt->sb.depth == 1)
		return true;

	users = atomic_read(&hctx->tags->active_queues);
	if (!users)
		return true;

	/*
	 * Allow at least some tags
	 */
	depth = max((bt->sb.depth + users - 1) / users, 4U);
	return atomic_read(&hctx->nr_active) < depth;
}

static int __bt_get(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
{
	if (!hctx_may_queue(hctx, bt))
		return -1;
	return __sbitmap_queue_get(bt);
}

static int bt_get(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt,
		  struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags)
{
	struct sbq_wait_state *ws;
	DEFINE_WAIT(wait);
	int tag;

	tag = __bt_get(hctx, bt);
	if (tag != -1)
		return tag;

	if (data->flags & BLK_MQ_REQ_NOWAIT)
		return -1;

	ws = bt_wait_ptr(bt, hctx);
	do {
		prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);

		tag = __bt_get(hctx, bt);
		if (tag != -1)
			break;

		/*
		 * We're out of tags on this hardware queue, kick any
		 * pending IO submits before going to sleep waiting for
		 * some to complete. Note that hctx can be NULL here for
		 * reserved tag allocation.
		 */
		if (hctx)
			blk_mq_run_hw_queue(hctx, false);

		/*
		 * Retry tag allocation after running the hardware queue,
		 * as running the queue may also have found completions.
		 */
		tag = __bt_get(hctx, bt);
		if (tag != -1)
			break;

		blk_mq_put_ctx(data->ctx);

		io_schedule();

		data->ctx = blk_mq_get_ctx(data->q);
		data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu);
		if (data->flags & BLK_MQ_REQ_RESERVED) {
			bt = &data->hctx->tags->breserved_tags;
		} else {
			hctx = data->hctx;
			bt = &hctx->tags->bitmap_tags;
		}
		finish_wait(&ws->wait, &wait);
		ws = bt_wait_ptr(bt, hctx);
	} while (1);

	finish_wait(&ws->wait, &wait);
	return tag;
}

static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
	int tag;

	tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
		     data->hctx->tags);
	if (tag >= 0)
		return tag + data->hctx->tags->nr_reserved_tags;

	return BLK_MQ_TAG_FAIL;
}

static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
{
	int tag;

	if (unlikely(!data->hctx->tags->nr_reserved_tags)) {
		WARN_ON_ONCE(1);
		return BLK_MQ_TAG_FAIL;
	}

	tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL,
		     data->hctx->tags);
	if (tag < 0)
		return BLK_MQ_TAG_FAIL;

	return tag;
}

unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
	if (data->flags & BLK_MQ_REQ_RESERVED)
		return __blk_mq_get_reserved_tag(data);
	return __blk_mq_get_tag(data);
}

void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
		    unsigned int tag)
{
	struct blk_mq_tags *tags = hctx->tags;

	if (tag >= tags->nr_reserved_tags) {
		const int real_tag = tag - tags->nr_reserved_tags;

		BUG_ON(real_tag >= tags->nr_tags);
		sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
	} else {
		BUG_ON(tag >= tags->nr_reserved_tags);
		sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
	}
}

struct bt_iter_data {
	struct blk_mq_hw_ctx *hctx;
	busy_iter_fn *fn;
	void *data;
	bool reserved;
};

static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{
	struct bt_iter_data *iter_data = data;
	struct blk_mq_hw_ctx *hctx = iter_data->hctx;
	struct blk_mq_tags *tags = hctx->tags;
	bool reserved = iter_data->reserved;
	struct request *rq;

	if (!reserved)
		bitnr += tags->nr_reserved_tags;
	rq = tags->rqs[bitnr];

	if (rq->q == hctx->queue)
		iter_data->fn(hctx, rq, iter_data->data, reserved);
	return true;
}

static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt,
			busy_iter_fn *fn, void *data, bool reserved)
{
	struct bt_iter_data iter_data = {
		.hctx = hctx,
		.fn = fn,
		.data = data,
		.reserved = reserved,
	};

	sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data);
}

struct bt_tags_iter_data {
	struct blk_mq_tags *tags;
	busy_tag_iter_fn *fn;
	void *data;
	bool reserved;
};

static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data)
{
	struct bt_tags_iter_data *iter_data = data;
	struct blk_mq_tags *tags = iter_data->tags;
	bool reserved = iter_data->reserved;
	struct request *rq;

	if (!reserved)
		bitnr += tags->nr_reserved_tags;
	rq = tags->rqs[bitnr];

	iter_data->fn(rq, iter_data->data, reserved);
	return true;
}

static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt,
			     busy_tag_iter_fn *fn, void *data, bool reserved)
{
	struct bt_tags_iter_data iter_data = {
		.tags = tags,
		.fn = fn,
		.data = data,
		.reserved = reserved,
	};

	if (tags->rqs)
		sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
}

static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
		busy_tag_iter_fn *fn, void *priv)
{
	if (tags->nr_reserved_tags)
		bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true);
	bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
}

void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
		busy_tag_iter_fn *fn, void *priv)
{
	int i;

	for (i = 0; i < tagset->nr_hw_queues; i++) {
		if (tagset->tags && tagset->tags[i])
			blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv);
	}
}
EXPORT_SYMBOL(blk_mq_tagset_busy_iter);

int blk_mq_reinit_tagset(struct blk_mq_tag_set *set)
{
	int i, j, ret = 0;

	if (!set->ops->reinit_request)
		goto out;

	for (i = 0; i < set->nr_hw_queues; i++) {
		struct blk_mq_tags *tags = set->tags[i];

		if (!tags)
			continue;

		for (j = 0; j < tags->nr_tags; j++) {
			if (!tags->rqs[j])
				continue;

			ret = set->ops->reinit_request(set->driver_data,
						tags->rqs[j]);
			if (ret)
				goto out;
		}
	}

out:
	return ret;
}
EXPORT_SYMBOL_GPL(blk_mq_reinit_tagset);

void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn,
		void *priv)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	/*
	 * __blk_mq_update_nr_hw_queues will update the nr_hw_queues and
	 * queue_hw_ctx after freeze the queue, so we use q_usage_counter
	 * to avoid race with it.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
		return;

	queue_for_each_hw_ctx(q, hctx, i) {
		struct blk_mq_tags *tags = hctx->tags;

		/*
		 * If not software queues are currently mapped to this
		 * hardware queue, there's nothing to check
		 */
		if (!blk_mq_hw_queue_mapped(hctx))
			continue;

		if (tags->nr_reserved_tags)
			bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
		bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
	}
	blk_queue_exit(q);
}

static unsigned int bt_unused_tags(const struct sbitmap_queue *bt)
{
	return bt->sb.depth - sbitmap_weight(&bt->sb);
}

static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
		    bool round_robin, int node)
{
	return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL,
				       node);
}

static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags,
						   int node, int alloc_policy)
{
	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;

	if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
		goto free_tags;
	if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
		     node))
		goto free_bitmap_tags;

	return tags;
free_bitmap_tags:
	sbitmap_queue_free(&tags->bitmap_tags);
free_tags:
	kfree(tags);
	return NULL;
}

struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
				     unsigned int reserved_tags,
				     int node, int alloc_policy)
{
	struct blk_mq_tags *tags;

	if (total_tags > BLK_MQ_TAG_MAX) {
		pr_err("blk-mq: tag depth too large\n");
		return NULL;
	}

	tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
	if (!tags)
		return NULL;

	tags->nr_tags = total_tags;
	tags->nr_reserved_tags = reserved_tags;

	return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
}

void blk_mq_free_tags(struct blk_mq_tags *tags)
{
	sbitmap_queue_free(&tags->bitmap_tags);
	sbitmap_queue_free(&tags->breserved_tags);
	kfree(tags);
}

int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
{
	tdepth -= tags->nr_reserved_tags;
	if (tdepth > tags->nr_tags)
		return -EINVAL;

	/*
	 * Don't need (or can't) update reserved tags here, they remain
	 * static and should never need resizing.
	 */
	sbitmap_queue_resize(&tags->bitmap_tags, tdepth);

	blk_mq_tag_wakeup_all(tags, false);
	return 0;
}

/**
 * blk_mq_unique_tag() - return a tag that is unique queue-wide
 * @rq: request for which to compute a unique tag
 *
 * The tag field in struct request is unique per hardware queue but not over
 * all hardware queues. Hence this function that returns a tag with the
 * hardware context index in the upper bits and the per hardware queue tag in
 * the lower bits.
 *
 * Note: When called for a request that is queued on a non-multiqueue request
 * queue, the hardware context index is set to zero.
 */
u32 blk_mq_unique_tag(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct blk_mq_hw_ctx *hctx;
	int hwq = 0;

	if (q->mq_ops) {
		hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
		hwq = hctx->queue_num;
	}

	return (hwq << BLK_MQ_UNIQUE_TAG_BITS) |
		(rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
}
EXPORT_SYMBOL(blk_mq_unique_tag);

ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
{
	char *orig_page = page;
	unsigned int free, res;

	if (!tags)
		return 0;

	page += sprintf(page, "nr_tags=%u, reserved_tags=%u, "
			"bits_per_word=%u\n",
			tags->nr_tags, tags->nr_reserved_tags,
			1U << tags->bitmap_tags.sb.shift);

	free = bt_unused_tags(&tags->bitmap_tags);
	res = bt_unused_tags(&tags->breserved_tags);

	page += sprintf(page, "nr_free=%u, nr_reserved=%u\n", free, res);
	page += sprintf(page, "active_queues=%u\n", atomic_read(&tags->active_queues));

	return page - orig_page;
}
Commit	Line	Data
75bb4625	1	/*
88459642 OS	2	* Tag allocation using scalable bitmaps. Uses active queue tracking to support
	3	* fairer distribution of tags between multiple submitters when a shared tag map
	4	* is used.
75bb4625 JA	5	*
	6	* Copyright (C) 2013-2014 Jens Axboe
	7	*/
320ae51f JA	8	#include <linux/kernel.h>
320ae51f JA	9	#include <linux/module.h>
320ae51f JA	10
	11	#include <linux/blk-mq.h>
	12	#include "blk.h"
	13	#include "blk-mq.h"
	14	#include "blk-mq-tag.h"
	15
320ae51f JA	16	bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
320ae51f JA	17	{
4bb659b1 JA	18	if (!tags)
	19	return true;
	20
88459642	21	return sbitmap_any_bit_clear(&tags->bitmap_tags.sb);
0d2602ca JA	22	}
	23
	24	/*
	25	* If a previously inactive queue goes active, bump the active user count.
	26	*/
	27	bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
	28	{
	29	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) &&
	30	!test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
	31	atomic_inc(&hctx->tags->active_queues);
	32
	33	return true;
	34	}
	35
	36	/*
aed3ea94	37	* Wakeup all potentially sleeping on tags
0d2602ca	38	*/
aed3ea94	39	void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
0d2602ca	40	{
88459642 OS	41	sbitmap_queue_wake_all(&tags->bitmap_tags);
	42	if (include_reserve)
	43	sbitmap_queue_wake_all(&tags->breserved_tags);
0d2602ca JA	44	}
0d2602ca JA	45
e3a2b3f9 JA	46	/*
	47	* If a previously busy queue goes inactive, potential waiters could now
	48	* be allowed to queue. Wake them up and check.
	49	*/
	50	void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
	51	{
	52	struct blk_mq_tags *tags = hctx->tags;
	53
	54	if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
	55	return;
	56
	57	atomic_dec(&tags->active_queues);
	58
aed3ea94	59	blk_mq_tag_wakeup_all(tags, false);
e3a2b3f9 JA	60	}
e3a2b3f9 JA	61
0d2602ca JA	62	/*
	63	* For shared tag users, we track the number of currently active users
	64	* and attempt to provide a fair share of the tag depth for each of them.
	65	*/
	66	static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx,
88459642	67	struct sbitmap_queue *bt)
0d2602ca JA	68	{
	69	unsigned int depth, users;
	70
	71	if (!hctx \|\| !(hctx->flags & BLK_MQ_F_TAG_SHARED))
	72	return true;
	73	if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state))
	74	return true;
	75
	76	/*
	77	* Don't try dividing an ant
	78	*/
88459642	79	if (bt->sb.depth == 1)
0d2602ca JA	80	return true;
	81
	82	users = atomic_read(&hctx->tags->active_queues);
	83	if (!users)
	84	return true;
	85
	86	/*
	87	* Allow at least some tags
	88	*/
88459642	89	depth = max((bt->sb.depth + users - 1) / users, 4U);
0d2602ca JA	90	return atomic_read(&hctx->nr_active) < depth;
	91	}
	92
f4a644db	93	static int __bt_get(struct blk_mq_hw_ctx hctx, struct sbitmap_queue bt)
4bb659b1	94	{
0d2602ca JA	95	if (!hctx_may_queue(hctx, bt))
0d2602ca JA	96	return -1;
f4a644db	97	return __sbitmap_queue_get(bt);
4bb659b1 JA	98	}
4bb659b1 JA	99
40aabb67 OS	100	static int bt_get(struct blk_mq_alloc_data data, struct sbitmap_queue bt,
40aabb67 OS	101	struct blk_mq_hw_ctx hctx, struct blk_mq_tags tags)
320ae51f	102	{
88459642	103	struct sbq_wait_state *ws;
4bb659b1	104	DEFINE_WAIT(wait);
320ae51f JA	105	int tag;
320ae51f JA	106
f4a644db	107	tag = __bt_get(hctx, bt);
4bb659b1 JA	108	if (tag != -1)
	109	return tag;
	110
6f3b0e8b	111	if (data->flags & BLK_MQ_REQ_NOWAIT)
4bb659b1 JA	112	return -1;
4bb659b1 JA	113
88459642	114	ws = bt_wait_ptr(bt, hctx);
4bb659b1	115	do {
88459642	116	prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);
4bb659b1	117
f4a644db	118	tag = __bt_get(hctx, bt);
4bb659b1 JA	119	if (tag != -1)
	120	break;
	121
b3223207 BVA	122	/*
	123	* We're out of tags on this hardware queue, kick any
	124	* pending IO submits before going to sleep waiting for
bc188d81 SB	125	* some to complete. Note that hctx can be NULL here for
bc188d81 SB	126	* reserved tag allocation.
b3223207	127	*/
bc188d81 SB	128	if (hctx)
bc188d81 SB	129	blk_mq_run_hw_queue(hctx, false);
b3223207	130
080ff351 JA	131	/*
	132	* Retry tag allocation after running the hardware queue,
	133	* as running the queue may also have found completions.
	134	*/
f4a644db	135	tag = __bt_get(hctx, bt);
080ff351 JA	136	if (tag != -1)
	137	break;
	138
cb96a42c ML	139	blk_mq_put_ctx(data->ctx);
cb96a42c ML	140
4bb659b1	141	io_schedule();
cb96a42c ML	142
cb96a42c ML	143	data->ctx = blk_mq_get_ctx(data->q);
7d7e0f90	144	data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu);
6f3b0e8b	145	if (data->flags & BLK_MQ_REQ_RESERVED) {
cb96a42c ML	146	bt = &data->hctx->tags->breserved_tags;
cb96a42c ML	147	} else {
cb96a42c ML	148	hctx = data->hctx;
	149	bt = &hctx->tags->bitmap_tags;
	150	}
88459642 OS	151	finish_wait(&ws->wait, &wait);
88459642 OS	152	ws = bt_wait_ptr(bt, hctx);
4bb659b1 JA	153	} while (1);
4bb659b1 JA	154
88459642	155	finish_wait(&ws->wait, &wait);
4bb659b1 JA	156	return tag;
	157	}
	158
cb96a42c	159	static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
4bb659b1 JA	160	{
	161	int tag;
	162
cb96a42c	163	tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
40aabb67	164	data->hctx->tags);
4bb659b1	165	if (tag >= 0)
cb96a42c	166	return tag + data->hctx->tags->nr_reserved_tags;
4bb659b1 JA	167
4bb659b1 JA	168	return BLK_MQ_TAG_FAIL;
320ae51f JA	169	}
320ae51f JA	170
cb96a42c	171	static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
320ae51f	172	{
40aabb67	173	int tag;
320ae51f	174
cb96a42c	175	if (unlikely(!data->hctx->tags->nr_reserved_tags)) {
320ae51f JA	176	WARN_ON_ONCE(1);
	177	return BLK_MQ_TAG_FAIL;
	178	}
	179
40aabb67 OS	180	tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL,
40aabb67 OS	181	data->hctx->tags);
320ae51f JA	182	if (tag < 0)
320ae51f JA	183	return BLK_MQ_TAG_FAIL;
4bb659b1	184
320ae51f JA	185	return tag;
	186	}
	187
cb96a42c	188	unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
320ae51f	189	{
6f3b0e8b CH	190	if (data->flags & BLK_MQ_REQ_RESERVED)
	191	return __blk_mq_get_reserved_tag(data);
	192	return __blk_mq_get_tag(data);
320ae51f JA	193	}
320ae51f JA	194
40aabb67 OS	195	void blk_mq_put_tag(struct blk_mq_hw_ctx hctx, struct blk_mq_ctx ctx,
40aabb67 OS	196	unsigned int tag)
320ae51f	197	{
0d2602ca JA	198	struct blk_mq_tags *tags = hctx->tags;
0d2602ca JA	199
4bb659b1 JA	200	if (tag >= tags->nr_reserved_tags) {
	201	const int real_tag = tag - tags->nr_reserved_tags;
	202
70114c39	203	BUG_ON(real_tag >= tags->nr_tags);
f4a644db	204	sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu);
70114c39 JA	205	} else {
70114c39 JA	206	BUG_ON(tag >= tags->nr_reserved_tags);
f4a644db	207	sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu);
70114c39	208	}
320ae51f JA	209	}
320ae51f JA	210
88459642 OS	211	struct bt_iter_data {
	212	struct blk_mq_hw_ctx *hctx;
	213	busy_iter_fn *fn;
	214	void *data;
	215	bool reserved;
	216	};
	217
	218	static bool bt_iter(struct sbitmap bitmap, unsigned int bitnr, void data)
320ae51f	219	{
88459642 OS	220	struct bt_iter_data *iter_data = data;
	221	struct blk_mq_hw_ctx *hctx = iter_data->hctx;
	222	struct blk_mq_tags *tags = hctx->tags;
	223	bool reserved = iter_data->reserved;
81481eb4	224	struct request *rq;
4bb659b1	225
88459642 OS	226	if (!reserved)
	227	bitnr += tags->nr_reserved_tags;
	228	rq = tags->rqs[bitnr];
4bb659b1	229
88459642 OS	230	if (rq->q == hctx->queue)
	231	iter_data->fn(hctx, rq, iter_data->data, reserved);
	232	return true;
	233	}
4bb659b1	234
88459642 OS	235	static void bt_for_each(struct blk_mq_hw_ctx hctx, struct sbitmap_queue bt,
	236	busy_iter_fn fn, void data, bool reserved)
	237	{
	238	struct bt_iter_data iter_data = {
	239	.hctx = hctx,
	240	.fn = fn,
	241	.data = data,
	242	.reserved = reserved,
	243	};
	244
	245	sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data);
320ae51f JA	246	}
320ae51f JA	247
88459642 OS	248	struct bt_tags_iter_data {
	249	struct blk_mq_tags *tags;
	250	busy_tag_iter_fn *fn;
	251	void *data;
	252	bool reserved;
	253	};
	254
	255	static bool bt_tags_iter(struct sbitmap bitmap, unsigned int bitnr, void data)
f26cdc85	256	{
88459642 OS	257	struct bt_tags_iter_data *iter_data = data;
	258	struct blk_mq_tags *tags = iter_data->tags;
	259	bool reserved = iter_data->reserved;
f26cdc85	260	struct request *rq;
f26cdc85	261
88459642 OS	262	if (!reserved)
	263	bitnr += tags->nr_reserved_tags;
	264	rq = tags->rqs[bitnr];
f26cdc85	265
88459642 OS	266	iter_data->fn(rq, iter_data->data, reserved);
	267	return true;
	268	}
	269
	270	static void bt_tags_for_each(struct blk_mq_tags tags, struct sbitmap_queue bt,
	271	busy_tag_iter_fn fn, void data, bool reserved)
	272	{
	273	struct bt_tags_iter_data iter_data = {
	274	.tags = tags,
	275	.fn = fn,
	276	.data = data,
	277	.reserved = reserved,
	278	};
	279
	280	if (tags->rqs)
	281	sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data);
f26cdc85 KB	282	}
f26cdc85 KB	283
e8f1e163 SG	284	static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags,
e8f1e163 SG	285	busy_tag_iter_fn fn, void priv)
f26cdc85 KB	286	{
f26cdc85 KB	287	if (tags->nr_reserved_tags)
88459642 OS	288	bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true);
88459642 OS	289	bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false);
f26cdc85	290	}
f26cdc85	291
e0489487 SG	292	void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
	293	busy_tag_iter_fn fn, void priv)
	294	{
	295	int i;
	296
	297	for (i = 0; i < tagset->nr_hw_queues; i++) {
	298	if (tagset->tags && tagset->tags[i])
	299	blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv);
	300	}
	301	}
	302	EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
	303
486cf989 SG	304	int blk_mq_reinit_tagset(struct blk_mq_tag_set *set)
	305	{
	306	int i, j, ret = 0;
	307
	308	if (!set->ops->reinit_request)
	309	goto out;
	310
	311	for (i = 0; i < set->nr_hw_queues; i++) {
	312	struct blk_mq_tags *tags = set->tags[i];
	313
d28046fb SG	314	if (!tags)
	315	continue;
	316
486cf989 SG	317	for (j = 0; j < tags->nr_tags; j++) {
	318	if (!tags->rqs[j])
	319	continue;
	320
	321	ret = set->ops->reinit_request(set->driver_data,
	322	tags->rqs[j]);
	323	if (ret)
	324	goto out;
	325	}
	326	}
	327
	328	out:
	329	return ret;
	330	}
	331	EXPORT_SYMBOL_GPL(blk_mq_reinit_tagset);
	332
0bf6cd5b	333	void blk_mq_queue_tag_busy_iter(struct request_queue q, busy_iter_fn fn,
81481eb4	334	void *priv)
320ae51f	335	{
0bf6cd5b CH	336	struct blk_mq_hw_ctx *hctx;
	337	int i;
	338
f530afb9 JW	339	/*
f530afb9 JW	340	* __blk_mq_update_nr_hw_queues will update the nr_hw_queues and
d4d74449 KB	341	* queue_hw_ctx after freeze the queue, so we use q_usage_counter
d4d74449 KB	342	* to avoid race with it.
f530afb9	343	*/
d4d74449	344	if (!percpu_ref_tryget(&q->q_usage_counter))
f530afb9	345	return;
0bf6cd5b CH	346
	347	queue_for_each_hw_ctx(q, hctx, i) {
	348	struct blk_mq_tags *tags = hctx->tags;
	349
	350	/*
	351	* If not software queues are currently mapped to this
	352	* hardware queue, there's nothing to check
	353	*/
	354	if (!blk_mq_hw_queue_mapped(hctx))
	355	continue;
	356
	357	if (tags->nr_reserved_tags)
88459642 OS	358	bt_for_each(hctx, &tags->breserved_tags, fn, priv, true);
88459642 OS	359	bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false);
4bb659b1	360	}
d4d74449	361	blk_queue_exit(q);
4bb659b1 JA	362	}
4bb659b1 JA	363
88459642	364	static unsigned int bt_unused_tags(const struct sbitmap_queue *bt)
e3a2b3f9	365	{
88459642	366	return bt->sb.depth - sbitmap_weight(&bt->sb);
e3a2b3f9 JA	367	}
e3a2b3f9 JA	368
f4a644db OS	369	static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
f4a644db OS	370	bool round_robin, int node)
4bb659b1	371	{
f4a644db OS	372	return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL,
f4a644db OS	373	node);
4bb659b1 JA	374	}
	375
	376	static struct blk_mq_tags blk_mq_init_bitmap_tags(struct blk_mq_tags tags,
24391c0d	377	int node, int alloc_policy)
4bb659b1 JA	378	{
4bb659b1 JA	379	unsigned int depth = tags->nr_tags - tags->nr_reserved_tags;
f4a644db	380	bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR;
4bb659b1	381
f4a644db	382	if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node))
88459642	383	goto free_tags;
f4a644db OS	384	if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin,
f4a644db OS	385	node))
88459642	386	goto free_bitmap_tags;
4bb659b1 JA	387
4bb659b1 JA	388	return tags;
88459642 OS	389	free_bitmap_tags:
	390	sbitmap_queue_free(&tags->bitmap_tags);
	391	free_tags:
4bb659b1 JA	392	kfree(tags);
	393	return NULL;
	394	}
	395
320ae51f	396	struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
24391c0d SL	397	unsigned int reserved_tags,
24391c0d SL	398	int node, int alloc_policy)
320ae51f	399	{
320ae51f	400	struct blk_mq_tags *tags;
320ae51f JA	401
	402	if (total_tags > BLK_MQ_TAG_MAX) {
	403	pr_err("blk-mq: tag depth too large\n");
	404	return NULL;
	405	}
	406
	407	tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
	408	if (!tags)
	409	return NULL;
	410
320ae51f JA	411	tags->nr_tags = total_tags;
320ae51f JA	412	tags->nr_reserved_tags = reserved_tags;
320ae51f	413
24391c0d	414	return blk_mq_init_bitmap_tags(tags, node, alloc_policy);
320ae51f JA	415	}
	416
	417	void blk_mq_free_tags(struct blk_mq_tags *tags)
	418	{
88459642 OS	419	sbitmap_queue_free(&tags->bitmap_tags);
88459642 OS	420	sbitmap_queue_free(&tags->breserved_tags);
320ae51f JA	421	kfree(tags);
	422	}
	423
e3a2b3f9 JA	424	int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
	425	{
	426	tdepth -= tags->nr_reserved_tags;
	427	if (tdepth > tags->nr_tags)
	428	return -EINVAL;
	429
	430	/*
	431	* Don't need (or can't) update reserved tags here, they remain
	432	* static and should never need resizing.
	433	*/
88459642 OS	434	sbitmap_queue_resize(&tags->bitmap_tags, tdepth);
88459642 OS	435
aed3ea94	436	blk_mq_tag_wakeup_all(tags, false);
e3a2b3f9 JA	437	return 0;
	438	}
	439
205fb5f5 BVA	440	/**
	441	* blk_mq_unique_tag() - return a tag that is unique queue-wide
	442	* @rq: request for which to compute a unique tag
	443	*
	444	* The tag field in struct request is unique per hardware queue but not over
	445	* all hardware queues. Hence this function that returns a tag with the
	446	* hardware context index in the upper bits and the per hardware queue tag in
	447	* the lower bits.
	448	*
	449	* Note: When called for a request that is queued on a non-multiqueue request
	450	* queue, the hardware context index is set to zero.
	451	*/
	452	u32 blk_mq_unique_tag(struct request *rq)
	453	{
	454	struct request_queue *q = rq->q;
	455	struct blk_mq_hw_ctx *hctx;
	456	int hwq = 0;
	457
	458	if (q->mq_ops) {
7d7e0f90	459	hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
205fb5f5 BVA	460	hwq = hctx->queue_num;
	461	}
	462
	463	return (hwq << BLK_MQ_UNIQUE_TAG_BITS) \|
	464	(rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
	465	}
	466	EXPORT_SYMBOL(blk_mq_unique_tag);
	467
320ae51f JA	468	ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags tags, char page)
	469	{
	470	char *orig_page = page;
4bb659b1	471	unsigned int free, res;
320ae51f JA	472
	473	if (!tags)
	474	return 0;
	475
59d13bf5 JA	476	page += sprintf(page, "nr_tags=%u, reserved_tags=%u, "
	477	"bits_per_word=%u\n",
	478	tags->nr_tags, tags->nr_reserved_tags,
88459642	479	1U << tags->bitmap_tags.sb.shift);
320ae51f	480
4bb659b1 JA	481	free = bt_unused_tags(&tags->bitmap_tags);
4bb659b1 JA	482	res = bt_unused_tags(&tags->breserved_tags);
320ae51f	483
4bb659b1	484	page += sprintf(page, "nr_free=%u, nr_reserved=%u\n", free, res);
0d2602ca	485	page += sprintf(page, "active_queues=%u\n", atomic_read(&tags->active_queues));
320ae51f JA	486
	487	return page - orig_page;
	488	}