blk-mq: move the cache friendly bitmap type of out blk-mq-tag

We will use it for the pending list in blk-mq core as well.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 8d526a3e02f6b951e9bf6ee729ff5225579db28a..03ce6a11ba791de1002ac79b72a3ce7eac12a8b7 100644 (file)
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -21,7 +21,7 @@ static bool bt_has_free_tags(struct blk_mq_bitmap_tags *bt)
        int i;
 
        for (i = 0; i < bt->map_nr; i++) {
-               struct blk_mq_bitmap *bm = &bt->map[i];
+               struct blk_align_bitmap *bm = &bt->map[i];
                int ret;
 
                ret = find_first_zero_bit(&bm->word, bm->depth);
@@ -40,7 +40,7 @@ bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
        return bt_has_free_tags(&tags->bitmap_tags);
 }
 
-static int __bt_get_word(struct blk_mq_bitmap *bm, unsigned int last_tag)
+static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag)
 {
        int tag, org_last_tag, end;
 
@@ -283,7 +283,7 @@ static void bt_for_each_free(struct blk_mq_bitmap_tags *bt,
        int i;
 
        for (i = 0; i < bt->map_nr; i++) {
-               struct blk_mq_bitmap *bm = &bt->map[i];
+               struct blk_align_bitmap *bm = &bt->map[i];
                int bit = 0;
 
                do {
@@ -323,7 +323,7 @@ static unsigned int bt_unused_tags(struct blk_mq_bitmap_tags *bt)
        unsigned int i, used;
 
        for (i = 0, used = 0; i < bt->map_nr; i++) {
-               struct blk_mq_bitmap *bm = &bt->map[i];
+               struct blk_align_bitmap *bm = &bt->map[i];
 
                used += bitmap_weight(&bm->word, bm->depth);
        }
@@ -361,7 +361,7 @@ static int bt_alloc(struct blk_mq_bitmap_tags *bt, unsigned int depth,
                }
 
                nr = ALIGN(depth, tags_per_word) / tags_per_word;
-               bt->map = kzalloc_node(nr * sizeof(struct blk_mq_bitmap),
+               bt->map = kzalloc_node(nr * sizeof(struct blk_align_bitmap),
                                                GFP_KERNEL, node);
                if (!bt->map)
                        return -ENOMEM;

diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 7aa9f06654898e8f0b1e7ffdc96ef907aeb6fa4f..9014269f39104152392d5fbf12f308757b1ef219 100644 (file)
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -1,6 +1,8 @@
 #ifndef INT_BLK_MQ_TAG_H
 #define INT_BLK_MQ_TAG_H
 
+#include "blk-mq.h"
+
 enum {
        BT_WAIT_QUEUES  = 8,
        BT_WAIT_BATCH   = 8,
@@ -14,18 +16,13 @@ struct bt_wait_state {
 #define TAG_TO_INDEX(bt, tag)  ((tag) >> (bt)->bits_per_word)
 #define TAG_TO_BIT(bt, tag)    ((tag) & ((1 << (bt)->bits_per_word) - 1))
 
-struct blk_mq_bitmap {
-       unsigned long word;
-       unsigned long depth;
-} ____cacheline_aligned_in_smp;
-
 struct blk_mq_bitmap_tags {
        unsigned int depth;
        unsigned int wake_cnt;
        unsigned int bits_per_word;
 
        unsigned int map_nr;
-       struct blk_mq_bitmap *map;
+       struct blk_align_bitmap *map;
 
        unsigned int wake_index;
        struct bt_wait_state *bs;

diff --git a/block/blk-mq.h b/block/blk-mq.h
index 97cfab9c092fdb3db32138137ba47e8c687e2a67..5e5a378962b7346fa019c37ef6ccbef9ac78b4ad 100644 (file)
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -52,4 +52,13 @@ void blk_mq_disable_hotplug(void);
 extern unsigned int *blk_mq_make_queue_map(struct blk_mq_tag_set *set);
 extern int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues);
 
+/*
+ * Basic implementation of sparser bitmap, allowing the user to spread
+ * the bits over more cachelines.
+ */
+struct blk_align_bitmap {
+       unsigned long word;
+       unsigned long depth;
+} ____cacheline_aligned_in_smp;
+
 #endif