projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
import PULS_20180308
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / block / nvme-core.c
CommitLineData
b60503ba
MW		 1/*
2 * NVM Express device driver
3 * Copyright (c) 2011, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 */
18
19#include <linux/nvme.h>
20#include <linux/bio.h>
8de05535 21#include <linux/bitops.h>
b60503ba 22#include <linux/blkdev.h>
fd63e9ce 23#include <linux/delay.h>
b60503ba
MW		 24#include <linux/errno.h>
25#include <linux/fs.h>
26#include <linux/genhd.h>
5aff9382 27#include <linux/idr.h>
b60503ba
MW		 28#include <linux/init.h>
29#include <linux/interrupt.h>
30#include <linux/io.h>
31#include <linux/kdev_t.h>
1fa6aead 32#include <linux/kthread.h>
b60503ba
MW		 33#include <linux/kernel.h>
34#include <linux/mm.h>
35#include <linux/module.h>
36#include <linux/moduleparam.h>
37#include <linux/pci.h>
be7b6275 38#include <linux/poison.h>
b60503ba
MW		 39#include <linux/sched.h>
40#include <linux/slab.h>
41#include <linux/types.h>
5d0f6131 42#include <scsi/sg.h>
797a796a
HM		 43#include <asm-generic/io-64-nonatomic-lo-hi.h>
44
b60503ba
MW		 45#define NVME_Q_DEPTH 1024
46#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
47#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
48#define NVME_MINORS 64
e85248e5 49#define ADMIN_TIMEOUT (60 * HZ)
b60503ba
MW		 50
51static int nvme_major;
52module_param(nvme_major, int, 0);
53
58ffacb5
MW		 54static int use_threaded_interrupts;
55module_param(use_threaded_interrupts, int, 0);
56
1fa6aead
MW		 57static DEFINE_SPINLOCK(dev_list_lock);
58static LIST_HEAD(dev_list);
59static struct task_struct *nvme_thread;
60
b60503ba
MW		 61/*
62 * An NVM Express queue. Each device has at least two (one for admin
63 * commands and one for I/O commands).
64 */
65struct nvme_queue {
66 struct device *q_dmadev;
091b6092 67 struct nvme_dev *dev;
b60503ba
MW		 68 spinlock_t q_lock;
69 struct nvme_command *sq_cmds;
70 volatile struct nvme_completion *cqes;
71 dma_addr_t sq_dma_addr;
72 dma_addr_t cq_dma_addr;
73 wait_queue_head_t sq_full;
1fa6aead 74 wait_queue_t sq_cong_wait;
b60503ba
MW		 75 struct bio_list sq_cong;
76 u32 __iomem *q_db;
77 u16 q_depth;
78 u16 cq_vector;
79 u16 sq_head;
80 u16 sq_tail;
81 u16 cq_head;
82123460 82 u16 cq_phase;
b60503ba
MW		 83 unsigned long cmdid_data[];
84};
85
86/*
87 * Check we didin't inadvertently grow the command struct
88 */
89static inline void _nvme_check_size(void)
90{
91 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
92 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
93 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
94 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
95 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
f8ebf840 96 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
b60503ba
MW		 97 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
98 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
99 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
100 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
6ecec745 101 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
b60503ba
MW		 102}
103
5c1281a3 104typedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
c2f5b650
MW		 105 struct nvme_completion *);
106
e85248e5 107struct nvme_cmd_info {
c2f5b650
MW		 108 nvme_completion_fn fn;
109 void *ctx;
e85248e5
MW		 110 unsigned long timeout;
111};
112
113static struct nvme_cmd_info *nvme_cmd_info(struct nvme_queue *nvmeq)
114{
115 return (void *)&nvmeq->cmdid_data[BITS_TO_LONGS(nvmeq->q_depth)];
116}
117
b60503ba 118/**
714a7a22
MW		 119 * alloc_cmdid() - Allocate a Command ID
120 * @nvmeq: The queue that will be used for this command
121 * @ctx: A pointer that will be passed to the handler
c2f5b650 122 * @handler: The function to call on completion
b60503ba
MW		 123 *
124 * Allocate a Command ID for a queue. The data passed in will
125 * be passed to the completion handler. This is implemented by using
126 * the bottom two bits of the ctx pointer to store the handler ID.
127 * Passing in a pointer that's not 4-byte aligned will cause a BUG.
128 * We can change this if it becomes a problem.
184d2944
MW		 129 *
130 * May be called with local interrupts disabled and the q_lock held,
131 * or with interrupts enabled and no locks held.
b60503ba 132 */
c2f5b650
MW		 133static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx,
134 nvme_completion_fn handler, unsigned timeout)
b60503ba 135{
e6d15f79 136 int depth = nvmeq->q_depth - 1;
e85248e5 137 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
b60503ba
MW		 138 int cmdid;
139
b60503ba
MW		 140 do {
141 cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
142 if (cmdid >= depth)
143 return -EBUSY;
144 } while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
145
c2f5b650
MW		 146 info[cmdid].fn = handler;
147 info[cmdid].ctx = ctx;
e85248e5 148 info[cmdid].timeout = jiffies + timeout;
b60503ba
MW		 149 return cmdid;
150}
151
152static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
c2f5b650 153 nvme_completion_fn handler, unsigned timeout)
b60503ba
MW		 154{
155 int cmdid;
156 wait_event_killable(nvmeq->sq_full,
e85248e5 157 (cmdid = alloc_cmdid(nvmeq, ctx, handler, timeout)) >= 0);
b60503ba
MW		 158 return (cmdid < 0) ? -EINTR : cmdid;
159}
160
c2f5b650
MW		 161/* Special values must be less than 0x1000 */
162#define CMD_CTX_BASE ((void *)POISON_POINTER_DELTA)
d2d87034
MW		 163#define CMD_CTX_CANCELLED (0x30C + CMD_CTX_BASE)
164#define CMD_CTX_COMPLETED (0x310 + CMD_CTX_BASE)
165#define CMD_CTX_INVALID (0x314 + CMD_CTX_BASE)
00df5cb4 166#define CMD_CTX_FLUSH (0x318 + CMD_CTX_BASE)
be7b6275 167
5c1281a3 168static void special_completion(struct nvme_dev *dev, void *ctx,
c2f5b650
MW		 169 struct nvme_completion *cqe)
170{
171 if (ctx == CMD_CTX_CANCELLED)
172 return;
173 if (ctx == CMD_CTX_FLUSH)
174 return;
175 if (ctx == CMD_CTX_COMPLETED) {
5c1281a3 176 dev_warn(&dev->pci_dev->dev,
c2f5b650
MW		 177 "completed id %d twice on queue %d\n",
178 cqe->command_id, le16_to_cpup(&cqe->sq_id));
179 return;
180 }
181 if (ctx == CMD_CTX_INVALID) {
5c1281a3 182 dev_warn(&dev->pci_dev->dev,
c2f5b650
MW		 183 "invalid id %d completed on queue %d\n",
184 cqe->command_id, le16_to_cpup(&cqe->sq_id));
185 return;
186 }
187
5c1281a3 188 dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
c2f5b650
MW		 189}
190
184d2944
MW		 191/*
192 * Called with local interrupts disabled and the q_lock held. May not sleep.
193 */
c2f5b650
MW		 194static void *free_cmdid(struct nvme_queue *nvmeq, int cmdid,
195 nvme_completion_fn *fn)
b60503ba 196{
c2f5b650 197 void *ctx;
e85248e5 198 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
b60503ba 199
c2f5b650
MW		 200 if (cmdid >= nvmeq->q_depth) {
201 *fn = special_completion;
48e3d398 202 return CMD_CTX_INVALID;
c2f5b650 203 }
859361a2
KB		 204 if (fn)
205 *fn = info[cmdid].fn;
c2f5b650
MW		 206 ctx = info[cmdid].ctx;
207 info[cmdid].fn = special_completion;
e85248e5 208 info[cmdid].ctx = CMD_CTX_COMPLETED;
b60503ba
MW		 209 clear_bit(cmdid, nvmeq->cmdid_data);
210 wake_up(&nvmeq->sq_full);
c2f5b650 211 return ctx;
b60503ba
MW		 212}
213
c2f5b650
MW		 214static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
215 nvme_completion_fn *fn)
3c0cf138 216{
c2f5b650 217 void *ctx;
e85248e5 218 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
c2f5b650
MW		 219 if (fn)
220 *fn = info[cmdid].fn;
221 ctx = info[cmdid].ctx;
222 info[cmdid].fn = special_completion;
e85248e5 223 info[cmdid].ctx = CMD_CTX_CANCELLED;
c2f5b650 224 return ctx;
3c0cf138
MW		 225}
226
5d0f6131 227struct nvme_queue *get_nvmeq(struct nvme_dev *dev)
b60503ba 228{
040a93b5 229 return dev->queues[get_cpu() + 1];
b60503ba
MW		 230}
231
5d0f6131 232void put_nvmeq(struct nvme_queue *nvmeq)
b60503ba 233{
1b23484b 234 put_cpu();
b60503ba
MW		 235}
236
237/**
714a7a22 238 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
b60503ba
MW		 239 * @nvmeq: The queue to use
240 * @cmd: The command to send
241 *
242 * Safe to use from interrupt context
243 */
244static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
245{
246 unsigned long flags;
247 u16 tail;
b60503ba
MW		 248 spin_lock_irqsave(&nvmeq->q_lock, flags);
249 tail = nvmeq->sq_tail;
250 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
b60503ba
MW		 251 if (++tail == nvmeq->q_depth)
252 tail = 0;
7547881d 253 writel(tail, nvmeq->q_db);
b60503ba
MW		 254 nvmeq->sq_tail = tail;
255 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
256
257 return 0;
258}
259
eca18b23 260static __le64 **iod_list(struct nvme_iod *iod)
e025344c 261{
eca18b23 262 return ((void *)iod) + iod->offset;
e025344c
SMM		 263}
264
eca18b23
MW		 265/*
266 * Will slightly overestimate the number of pages needed. This is OK
267 * as it only leads to a small amount of wasted memory for the lifetime of
268 * the I/O.
269 */
270static int nvme_npages(unsigned size)
271{
272 unsigned nprps = DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE);
273 return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
274}
b60503ba 275
eca18b23
MW		 276static struct nvme_iod *
277nvme_alloc_iod(unsigned nseg, unsigned nbytes, gfp_t gfp)
b60503ba 278{
eca18b23
MW		 279 struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
280 sizeof(__le64 *) * nvme_npages(nbytes) +
281 sizeof(struct scatterlist) * nseg, gfp);
282
283 if (iod) {
284 iod->offset = offsetof(struct nvme_iod, sg[nseg]);
285 iod->npages = -1;
286 iod->length = nbytes;
2b196034 287 iod->nents = 0;
eca18b23
MW		 288 }
289
290 return iod;
b60503ba
MW		 291}
292
5d0f6131 293void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
b60503ba 294{
eca18b23
MW		 295 const int last_prp = PAGE_SIZE / 8 - 1;
296 int i;
297 __le64 **list = iod_list(iod);
298 dma_addr_t prp_dma = iod->first_dma;
299
300 if (iod->npages == 0)
301 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
302 for (i = 0; i < iod->npages; i++) {
303 __le64 *prp_list = list[i];
304 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
305 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
306 prp_dma = next_prp_dma;
307 }
308 kfree(iod);
b60503ba
MW		 309}
310
5c1281a3 311static void bio_completion(struct nvme_dev *dev, void *ctx,
b60503ba
MW		 312 struct nvme_completion *cqe)
313{
eca18b23
MW		 314 struct nvme_iod *iod = ctx;
315 struct bio *bio = iod->private;
b60503ba
MW		 316 u16 status = le16_to_cpup(&cqe->status) >> 1;
317
2b196034
KB		 318 if (iod->nents)
319 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
b60503ba 320 bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
eca18b23 321 nvme_free_iod(dev, iod);
427e9708 322 if (status)
1ad2f893 323 bio_endio(bio, -EIO);
427e9708 324 else
1ad2f893 325 bio_endio(bio, 0);
b60503ba
MW		 326}
327
184d2944 328/* length is in bytes. gfp flags indicates whether we may sleep. */
5d0f6131
VV		 329int nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
330 struct nvme_iod *iod, int total_len, gfp_t gfp)
ff22b54f 331{
99802a7a 332 struct dma_pool *pool;
eca18b23
MW		 333 int length = total_len;
334 struct scatterlist *sg = iod->sg;
ff22b54f
MW		 335 int dma_len = sg_dma_len(sg);
336 u64 dma_addr = sg_dma_address(sg);
337 int offset = offset_in_page(dma_addr);
e025344c 338 __le64 *prp_list;
eca18b23 339 __le64 **list = iod_list(iod);
e025344c 340 dma_addr_t prp_dma;
eca18b23 341 int nprps, i;
ff22b54f
MW		 342
343 cmd->prp1 = cpu_to_le64(dma_addr);
344 length -= (PAGE_SIZE - offset);
345 if (length <= 0)
eca18b23 346 return total_len;
ff22b54f
MW		 347
348 dma_len -= (PAGE_SIZE - offset);
349 if (dma_len) {
350 dma_addr += (PAGE_SIZE - offset);
351 } else {
352 sg = sg_next(sg);
353 dma_addr = sg_dma_address(sg);
354 dma_len = sg_dma_len(sg);
355 }
356
357 if (length <= PAGE_SIZE) {
358 cmd->prp2 = cpu_to_le64(dma_addr);
eca18b23 359 return total_len;
e025344c
SMM		 360 }
361
362 nprps = DIV_ROUND_UP(length, PAGE_SIZE);
99802a7a
MW		 363 if (nprps <= (256 / 8)) {
364 pool = dev->prp_small_pool;
eca18b23 365 iod->npages = 0;
99802a7a
MW		 366 } else {
367 pool = dev->prp_page_pool;
eca18b23 368 iod->npages = 1;
99802a7a
MW		 369 }
370
b77954cb
MW		 371 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
372 if (!prp_list) {
373 cmd->prp2 = cpu_to_le64(dma_addr);
eca18b23
MW		 374 iod->npages = -1;
375 return (total_len - length) + PAGE_SIZE;
b77954cb 376 }
eca18b23
MW		 377 list[0] = prp_list;
378 iod->first_dma = prp_dma;
e025344c
SMM		 379 cmd->prp2 = cpu_to_le64(prp_dma);
380 i = 0;
381 for (;;) {
7523d834 382 if (i == PAGE_SIZE / 8) {
e025344c 383 __le64 *old_prp_list = prp_list;
b77954cb 384 prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
eca18b23
MW		 385 if (!prp_list)
386 return total_len - length;
387 list[iod->npages++] = prp_list;
7523d834
MW		 388 prp_list[0] = old_prp_list[i - 1];
389 old_prp_list[i - 1] = cpu_to_le64(prp_dma);
390 i = 1;
e025344c
SMM		 391 }
392 prp_list[i++] = cpu_to_le64(dma_addr);
393 dma_len -= PAGE_SIZE;
394 dma_addr += PAGE_SIZE;
395 length -= PAGE_SIZE;
396 if (length <= 0)
397 break;
398 if (dma_len > 0)
399 continue;
400 BUG_ON(dma_len < 0);
401 sg = sg_next(sg);
402 dma_addr = sg_dma_address(sg);
403 dma_len = sg_dma_len(sg);
ff22b54f
MW		 404 }
405
eca18b23 406 return total_len;
ff22b54f
MW		 407}
408
427e9708
KB		 409struct nvme_bio_pair {
410 struct bio b1, b2, *parent;
411 struct bio_vec *bv1, *bv2;
412 int err;
413 atomic_t cnt;
414};
415
416static void nvme_bio_pair_endio(struct bio *bio, int err)
417{
418 struct nvme_bio_pair *bp = bio->bi_private;
419
420 if (err)
421 bp->err = err;
422
423 if (atomic_dec_and_test(&bp->cnt)) {
424 bio_endio(bp->parent, bp->err);
425 if (bp->bv1)
426 kfree(bp->bv1);
427 if (bp->bv2)
428 kfree(bp->bv2);
429 kfree(bp);
430 }
431}
432
433static struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx,
434 int len, int offset)
435{
436 struct nvme_bio_pair *bp;
437
438 BUG_ON(len > bio->bi_size);
439 BUG_ON(idx > bio->bi_vcnt);
440
441 bp = kmalloc(sizeof(*bp), GFP_ATOMIC);
442 if (!bp)
443 return NULL;
444 bp->err = 0;
445
446 bp->b1 = *bio;
447 bp->b2 = *bio;
448
449 bp->b1.bi_size = len;
450 bp->b2.bi_size -= len;
451 bp->b1.bi_vcnt = idx;
452 bp->b2.bi_idx = idx;
453 bp->b2.bi_sector += len >> 9;
454
455 if (offset) {
456 bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
457 GFP_ATOMIC);
458 if (!bp->bv1)
459 goto split_fail_1;
460
461 bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
462 GFP_ATOMIC);
463 if (!bp->bv2)
464 goto split_fail_2;
465
466 memcpy(bp->bv1, bio->bi_io_vec,
467 bio->bi_max_vecs * sizeof(struct bio_vec));
468 memcpy(bp->bv2, bio->bi_io_vec,
469 bio->bi_max_vecs * sizeof(struct bio_vec));
470
471 bp->b1.bi_io_vec = bp->bv1;
472 bp->b2.bi_io_vec = bp->bv2;
473 bp->b2.bi_io_vec[idx].bv_offset += offset;
474 bp->b2.bi_io_vec[idx].bv_len -= offset;
475 bp->b1.bi_io_vec[idx].bv_len = offset;
476 bp->b1.bi_vcnt++;
477 } else
478 bp->bv1 = bp->bv2 = NULL;
479
480 bp->b1.bi_private = bp;
481 bp->b2.bi_private = bp;
482
483 bp->b1.bi_end_io = nvme_bio_pair_endio;
484 bp->b2.bi_end_io = nvme_bio_pair_endio;
485
486 bp->parent = bio;
487 atomic_set(&bp->cnt, 2);
488
489 return bp;
490
491 split_fail_2:
492 kfree(bp->bv1);
493 split_fail_1:
494 kfree(bp);
495 return NULL;
496}
497
498static int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
499 int idx, int len, int offset)
500{
501 struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset);
502 if (!bp)
503 return -ENOMEM;
504
505 if (bio_list_empty(&nvmeq->sq_cong))
506 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
507 bio_list_add(&nvmeq->sq_cong, &bp->b1);
508 bio_list_add(&nvmeq->sq_cong, &bp->b2);
509
510 return 0;
511}
512
1ad2f893
MW		 513/* NVMe scatterlists require no holes in the virtual address */
514#define BIOVEC_NOT_VIRT_MERGEABLE(vec1, vec2) ((vec2)->bv_offset || \
515 (((vec1)->bv_offset + (vec1)->bv_len) % PAGE_SIZE))
516
427e9708 517static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
b60503ba
MW		 518 struct bio *bio, enum dma_data_direction dma_dir, int psegs)
519{
76830840
MW		 520 struct bio_vec *bvec, *bvprv = NULL;
521 struct scatterlist *sg = NULL;
159b67d7
KB		 522 int i, length = 0, nsegs = 0, split_len = bio->bi_size;
523
524 if (nvmeq->dev->stripe_size)
525 split_len = nvmeq->dev->stripe_size -
526 ((bio->bi_sector << 9) & (nvmeq->dev->stripe_size - 1));
b60503ba 527
eca18b23 528 sg_init_table(iod->sg, psegs);
b60503ba 529 bio_for_each_segment(bvec, bio, i) {
76830840
MW		 530 if (bvprv && BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) {
531 sg->length += bvec->bv_len;
532 } else {
1ad2f893 533 if (bvprv && BIOVEC_NOT_VIRT_MERGEABLE(bvprv, bvec))
427e9708
KB		 534 return nvme_split_and_submit(bio, nvmeq, i,
535 length, 0);
536
eca18b23 537 sg = sg ? sg + 1 : iod->sg;
76830840
MW		 538 sg_set_page(sg, bvec->bv_page, bvec->bv_len,
539 bvec->bv_offset);
540 nsegs++;
541 }
159b67d7
KB		 542
543 if (split_len - length < bvec->bv_len)
544 return nvme_split_and_submit(bio, nvmeq, i, split_len,
545 split_len - length);
1ad2f893 546 length += bvec->bv_len;
76830840 547 bvprv = bvec;
b60503ba 548 }
eca18b23 549 iod->nents = nsegs;
76830840 550 sg_mark_end(sg);
427e9708 551 if (dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir) == 0)
1ad2f893 552 return -ENOMEM;
427e9708 553
159b67d7 554 BUG_ON(length != bio->bi_size);
1ad2f893 555 return length;
b60503ba
MW		 556}
557
0e5e4f0e
KB		 558/*
559 * We reuse the small pool to allocate the 16-byte range here as it is not
560 * worth having a special pool for these or additional cases to handle freeing
561 * the iod.
562 */
563static int nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
564 struct bio *bio, struct nvme_iod *iod, int cmdid)
565{
566 struct nvme_dsm_range *range;
567 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
568
569 range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
570 &iod->first_dma);
571 if (!range)
572 return -ENOMEM;
573
574 iod_list(iod)[0] = (__le64 *)range;
575 iod->npages = 0;
576
577 range->cattr = cpu_to_le32(0);
578 range->nlb = cpu_to_le32(bio->bi_size >> ns->lba_shift);
063cc6d5 579 range->slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
0e5e4f0e
KB		 580
581 memset(cmnd, 0, sizeof(*cmnd));
582 cmnd->dsm.opcode = nvme_cmd_dsm;
583 cmnd->dsm.command_id = cmdid;
584 cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
585 cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
586 cmnd->dsm.nr = 0;
587 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
588
589 if (++nvmeq->sq_tail == nvmeq->q_depth)
590 nvmeq->sq_tail = 0;
591 writel(nvmeq->sq_tail, nvmeq->q_db);
592
593 return 0;
594}
595
00df5cb4
MW		 596static int nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
597 int cmdid)
598{
599 struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
600
601 memset(cmnd, 0, sizeof(*cmnd));
602 cmnd->common.opcode = nvme_cmd_flush;
603 cmnd->common.command_id = cmdid;
604 cmnd->common.nsid = cpu_to_le32(ns->ns_id);
605
606 if (++nvmeq->sq_tail == nvmeq->q_depth)
607 nvmeq->sq_tail = 0;
608 writel(nvmeq->sq_tail, nvmeq->q_db);
609
610 return 0;
611}
612
5d0f6131 613int nvme_submit_flush_data(struct nvme_queue *nvmeq, struct nvme_ns *ns)
00df5cb4
MW		 614{
615 int cmdid = alloc_cmdid(nvmeq, (void *)CMD_CTX_FLUSH,
ff976d72 616 special_completion, NVME_IO_TIMEOUT);
00df5cb4
MW		 617 if (unlikely(cmdid < 0))
618 return cmdid;
619
620 return nvme_submit_flush(nvmeq, ns, cmdid);
621}
622
184d2944
MW		 623/*
624 * Called with local interrupts disabled and the q_lock held. May not sleep.
625 */
b60503ba
MW		 626static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
627 struct bio *bio)
628{
ff22b54f 629 struct nvme_command *cmnd;
eca18b23 630 struct nvme_iod *iod;
b60503ba 631 enum dma_data_direction dma_dir;
1287dabd 632 int cmdid, length, result;
b60503ba
MW		 633 u16 control;
634 u32 dsmgmt;
b60503ba
MW		 635 int psegs = bio_phys_segments(ns->queue, bio);
636
00df5cb4
MW		 637 if ((bio->bi_rw & REQ_FLUSH) && psegs) {
638 result = nvme_submit_flush_data(nvmeq, ns);
639 if (result)
640 return result;
641 }
642
1287dabd 643 result = -ENOMEM;
eca18b23
MW		 644 iod = nvme_alloc_iod(psegs, bio->bi_size, GFP_ATOMIC);
645 if (!iod)
eeee3226 646 goto nomem;
eca18b23 647 iod->private = bio;
b60503ba 648
eeee3226 649 result = -EBUSY;
ff976d72 650 cmdid = alloc_cmdid(nvmeq, iod, bio_completion, NVME_IO_TIMEOUT);
b60503ba 651 if (unlikely(cmdid < 0))
eca18b23 652 goto free_iod;
b60503ba 653
0e5e4f0e
KB		 654 if (bio->bi_rw & REQ_DISCARD) {
655 result = nvme_submit_discard(nvmeq, ns, bio, iod, cmdid);
656 if (result)
657 goto free_cmdid;
658 return result;
659 }
00df5cb4
MW		 660 if ((bio->bi_rw & REQ_FLUSH) && !psegs)
661 return nvme_submit_flush(nvmeq, ns, cmdid);
662
b60503ba
MW		 663 control = 0;
664 if (bio->bi_rw & REQ_FUA)
665 control |= NVME_RW_FUA;
666 if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
667 control |= NVME_RW_LR;
668
669 dsmgmt = 0;
670 if (bio->bi_rw & REQ_RAHEAD)
671 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
672
ff22b54f 673 cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
b60503ba 674
b8deb62c 675 memset(cmnd, 0, sizeof(*cmnd));
b60503ba 676 if (bio_data_dir(bio)) {
ff22b54f 677 cmnd->rw.opcode = nvme_cmd_write;
b60503ba
MW		 678 dma_dir = DMA_TO_DEVICE;
679 } else {
ff22b54f 680 cmnd->rw.opcode = nvme_cmd_read;
b60503ba
MW		 681 dma_dir = DMA_FROM_DEVICE;
682 }
683
427e9708
KB		 684 result = nvme_map_bio(nvmeq, iod, bio, dma_dir, psegs);
685 if (result <= 0)
859361a2 686 goto free_cmdid;
1ad2f893 687 length = result;
b60503ba 688
ff22b54f
MW		 689 cmnd->rw.command_id = cmdid;
690 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
eca18b23
MW		 691 length = nvme_setup_prps(nvmeq->dev, &cmnd->common, iod, length,
692 GFP_ATOMIC);
063cc6d5 693 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, bio->bi_sector));
1ad2f893 694 cmnd->rw.length = cpu_to_le16((length >> ns->lba_shift) - 1);
ff22b54f
MW		 695 cmnd->rw.control = cpu_to_le16(control);
696 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
b60503ba 697
b60503ba
MW		 698 if (++nvmeq->sq_tail == nvmeq->q_depth)
699 nvmeq->sq_tail = 0;
7547881d 700 writel(nvmeq->sq_tail, nvmeq->q_db);
b60503ba 701
1974b1ae
MW		 702 return 0;
703
859361a2
KB		 704 free_cmdid:
705 free_cmdid(nvmeq, cmdid, NULL);
eca18b23
MW		 706 free_iod:
707 nvme_free_iod(nvmeq->dev, iod);
eeee3226
MW		 708 nomem:
709 return result;
b60503ba
MW		 710}
711
93c3d65b 712static void nvme_make_request(struct request_queue *q, struct bio *bio)
b60503ba
MW		 713{
714 struct nvme_ns *ns = q->queuedata;
040a93b5 715 struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
eeee3226
MW		 716 int result = -EBUSY;
717
718 spin_lock_irq(&nvmeq->q_lock);
719 if (bio_list_empty(&nvmeq->sq_cong))
720 result = nvme_submit_bio_queue(nvmeq, ns, bio);
721 if (unlikely(result)) {
722 if (bio_list_empty(&nvmeq->sq_cong))
723 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
b60503ba
MW		 724 bio_list_add(&nvmeq->sq_cong, bio);
725 }
eeee3226
MW		 726
727 spin_unlock_irq(&nvmeq->q_lock);
b60503ba 728 put_nvmeq(nvmeq);
b60503ba
MW		 729}
730
b60503ba
MW		 731static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq)
732{
82123460 733 u16 head, phase;
b60503ba 734
b60503ba 735 head = nvmeq->cq_head;
82123460 736 phase = nvmeq->cq_phase;
b60503ba
MW		 737
738 for (;;) {
c2f5b650
MW		 739 void *ctx;
740 nvme_completion_fn fn;
b60503ba 741 struct nvme_completion cqe = nvmeq->cqes[head];
82123460 742 if ((le16_to_cpu(cqe.status) & 1) != phase)
b60503ba
MW		 743 break;
744 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
745 if (++head == nvmeq->q_depth) {
746 head = 0;
82123460 747 phase = !phase;
b60503ba
MW		 748 }
749
c2f5b650 750 ctx = free_cmdid(nvmeq, cqe.command_id, &fn);
5c1281a3 751 fn(nvmeq->dev, ctx, &cqe);
b60503ba
MW		 752 }
753
754 /* If the controller ignores the cq head doorbell and continuously
755 * writes to the queue, it is theoretically possible to wrap around
756 * the queue twice and mistakenly return IRQ_NONE. Linux only
757 * requires that 0.1% of your interrupts are handled, so this isn't
758 * a big problem.
759 */
82123460 760 if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
b60503ba
MW		 761 return IRQ_NONE;
762
f1938f6e 763 writel(head, nvmeq->q_db + (1 << nvmeq->dev->db_stride));
b60503ba 764 nvmeq->cq_head = head;
82123460 765 nvmeq->cq_phase = phase;
b60503ba
MW		 766
767 return IRQ_HANDLED;
768}
769
770static irqreturn_t nvme_irq(int irq, void *data)
58ffacb5
MW		 771{
772 irqreturn_t result;
773 struct nvme_queue *nvmeq = data;
774 spin_lock(&nvmeq->q_lock);
775 result = nvme_process_cq(nvmeq);
776 spin_unlock(&nvmeq->q_lock);
777 return result;
778}
779
780static irqreturn_t nvme_irq_check(int irq, void *data)
781{
782 struct nvme_queue *nvmeq = data;
783 struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
784 if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
785 return IRQ_NONE;
786 return IRQ_WAKE_THREAD;
787}
788
3c0cf138
MW		 789static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
790{
791 spin_lock_irq(&nvmeq->q_lock);
c2f5b650 792 cancel_cmdid(nvmeq, cmdid, NULL);
3c0cf138
MW		 793 spin_unlock_irq(&nvmeq->q_lock);
794}
795
c2f5b650
MW		 796struct sync_cmd_info {
797 struct task_struct *task;
798 u32 result;
799 int status;
800};
801
5c1281a3 802static void sync_completion(struct nvme_dev *dev, void *ctx,
c2f5b650
MW		 803 struct nvme_completion *cqe)
804{
805 struct sync_cmd_info *cmdinfo = ctx;
806 cmdinfo->result = le32_to_cpup(&cqe->result);
807 cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
808 wake_up_process(cmdinfo->task);
809}
810
b60503ba
MW		 811/*
812 * Returns 0 on success. If the result is negative, it's a Linux error code;
813 * if the result is positive, it's an NVM Express status code
814 */
5d0f6131
VV		 815int nvme_submit_sync_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
816 u32 *result, unsigned timeout)
b60503ba
MW		 817{
818 int cmdid;
819 struct sync_cmd_info cmdinfo;
820
821 cmdinfo.task = current;
822 cmdinfo.status = -EINTR;
823
c2f5b650 824 cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion,
e85248e5 825 timeout);
b60503ba
MW		 826 if (cmdid < 0)
827 return cmdid;
828 cmd->common.command_id = cmdid;
829
3c0cf138
MW		 830 set_current_state(TASK_KILLABLE);
831 nvme_submit_cmd(nvmeq, cmd);
78f8d257 832 schedule_timeout(timeout);
b60503ba 833
3c0cf138
MW		 834 if (cmdinfo.status == -EINTR) {
835 nvme_abort_command(nvmeq, cmdid);
836 return -EINTR;
837 }
838
b60503ba
MW		 839 if (result)
840 *result = cmdinfo.result;
841
842 return cmdinfo.status;
843}
844
5d0f6131 845int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
b60503ba
MW		 846 u32 *result)
847{
e85248e5 848 return nvme_submit_sync_cmd(dev->queues[0], cmd, result, ADMIN_TIMEOUT);
b60503ba
MW		 849}
850
851static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
852{
853 int status;
854 struct nvme_command c;
855
856 memset(&c, 0, sizeof(c));
857 c.delete_queue.opcode = opcode;
858 c.delete_queue.qid = cpu_to_le16(id);
859
860 status = nvme_submit_admin_cmd(dev, &c, NULL);
861 if (status)
862 return -EIO;
863 return 0;
864}
865
866static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
867 struct nvme_queue *nvmeq)
868{
869 int status;
870 struct nvme_command c;
871 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
872
873 memset(&c, 0, sizeof(c));
874 c.create_cq.opcode = nvme_admin_create_cq;
875 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
876 c.create_cq.cqid = cpu_to_le16(qid);
877 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
878 c.create_cq.cq_flags = cpu_to_le16(flags);
879 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
880
881 status = nvme_submit_admin_cmd(dev, &c, NULL);
882 if (status)
883 return -EIO;
884 return 0;
885}
886
887static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
888 struct nvme_queue *nvmeq)
889{
890 int status;
891 struct nvme_command c;
892 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
893
894 memset(&c, 0, sizeof(c));
895 c.create_sq.opcode = nvme_admin_create_sq;
896 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
897 c.create_sq.sqid = cpu_to_le16(qid);
898 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
899 c.create_sq.sq_flags = cpu_to_le16(flags);
900 c.create_sq.cqid = cpu_to_le16(qid);
901
902 status = nvme_submit_admin_cmd(dev, &c, NULL);
903 if (status)
904 return -EIO;
905 return 0;
906}
907
908static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
909{
910 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
911}
912
913static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
914{
915 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
916}
917
5d0f6131 918int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
bc5fc7e4
MW		 919 dma_addr_t dma_addr)
920{
921 struct nvme_command c;
922
923 memset(&c, 0, sizeof(c));
924 c.identify.opcode = nvme_admin_identify;
925 c.identify.nsid = cpu_to_le32(nsid);
926 c.identify.prp1 = cpu_to_le64(dma_addr);
927 c.identify.cns = cpu_to_le32(cns);
928
929 return nvme_submit_admin_cmd(dev, &c, NULL);
930}
931
5d0f6131 932int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
08df1e05 933 dma_addr_t dma_addr, u32 *result)
bc5fc7e4
MW		 934{
935 struct nvme_command c;
936
937 memset(&c, 0, sizeof(c));
938 c.features.opcode = nvme_admin_get_features;
a42cecce 939 c.features.nsid = cpu_to_le32(nsid);
bc5fc7e4
MW		 940 c.features.prp1 = cpu_to_le64(dma_addr);
941 c.features.fid = cpu_to_le32(fid);
bc5fc7e4 942
08df1e05 943 return nvme_submit_admin_cmd(dev, &c, result);
df348139
MW		 944}
945
5d0f6131
VV		 946int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
947 dma_addr_t dma_addr, u32 *result)
df348139
MW		 948{
949 struct nvme_command c;
950
951 memset(&c, 0, sizeof(c));
952 c.features.opcode = nvme_admin_set_features;
953 c.features.prp1 = cpu_to_le64(dma_addr);
954 c.features.fid = cpu_to_le32(fid);
955 c.features.dword11 = cpu_to_le32(dword11);
956
bc5fc7e4
MW		 957 return nvme_submit_admin_cmd(dev, &c, result);
958}
959
a09115b2
MW		 960/**
961 * nvme_cancel_ios - Cancel outstanding I/Os
962 * @queue: The queue to cancel I/Os on
963 * @timeout: True to only cancel I/Os which have timed out
964 */
965static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
966{
967 int depth = nvmeq->q_depth - 1;
968 struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
969 unsigned long now = jiffies;
970 int cmdid;
971
972 for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
973 void *ctx;
974 nvme_completion_fn fn;
975 static struct nvme_completion cqe = {
af2d9ca7 976 .status = cpu_to_le16(NVME_SC_ABORT_REQ << 1),
a09115b2
MW		 977 };
978
979 if (timeout && !time_after(now, info[cmdid].timeout))
980 continue;
053ab702
KB		 981 if (info[cmdid].ctx == CMD_CTX_CANCELLED)
982 continue;
a09115b2
MW		 983 dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
984 ctx = cancel_cmdid(nvmeq, cmdid, &fn);
985 fn(nvmeq->dev, ctx, &cqe);
986 }
987}
988
9e866774
MW		 989static void nvme_free_queue_mem(struct nvme_queue *nvmeq)
990{
991 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
992 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
993 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
994 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
995 kfree(nvmeq);
996}
997
b60503ba
MW		 998static void nvme_free_queue(struct nvme_dev *dev, int qid)
999{
1000 struct nvme_queue *nvmeq = dev->queues[qid];
aba2080f 1001 int vector = dev->entry[nvmeq->cq_vector].vector;
b60503ba 1002
a09115b2
MW		 1003 spin_lock_irq(&nvmeq->q_lock);
1004 nvme_cancel_ios(nvmeq, false);
3295874b
KB		 1005 while (bio_list_peek(&nvmeq->sq_cong)) {
1006 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1007 bio_endio(bio, -EIO);
1008 }
a09115b2
MW		 1009 spin_unlock_irq(&nvmeq->q_lock);
1010
aba2080f
MW		 1011 irq_set_affinity_hint(vector, NULL);
1012 free_irq(vector, nvmeq);
b60503ba
MW		 1013
1014 /* Don't tell the adapter to delete the admin queue */
1015 if (qid) {
1016 adapter_delete_sq(dev, qid);
1017 adapter_delete_cq(dev, qid);
1018 }
1019
9e866774 1020 nvme_free_queue_mem(nvmeq);
b60503ba
MW		 1021}
1022
1023static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1024 int depth, int vector)
1025{
1026 struct device *dmadev = &dev->pci_dev->dev;
a0cadb85
KB		 1027 unsigned extra = DIV_ROUND_UP(depth, 8) + (depth *
1028 sizeof(struct nvme_cmd_info));
b60503ba
MW		 1029 struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
1030 if (!nvmeq)
1031 return NULL;
1032
1033 nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
1034 &nvmeq->cq_dma_addr, GFP_KERNEL);
1035 if (!nvmeq->cqes)
1036 goto free_nvmeq;
1037 memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
1038
1039 nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
1040 &nvmeq->sq_dma_addr, GFP_KERNEL);
1041 if (!nvmeq->sq_cmds)
1042 goto free_cqdma;
1043
1044 nvmeq->q_dmadev = dmadev;
091b6092 1045 nvmeq->dev = dev;
b60503ba
MW		 1046 spin_lock_init(&nvmeq->q_lock);
1047 nvmeq->cq_head = 0;
82123460 1048 nvmeq->cq_phase = 1;
b60503ba 1049 init_waitqueue_head(&nvmeq->sq_full);
1fa6aead 1050 init_waitqueue_entry(&nvmeq->sq_cong_wait, nvme_thread);
b60503ba 1051 bio_list_init(&nvmeq->sq_cong);
f1938f6e 1052 nvmeq->q_db = &dev->dbs[qid << (dev->db_stride + 1)];
b60503ba
MW		 1053 nvmeq->q_depth = depth;
1054 nvmeq->cq_vector = vector;
1055
1056 return nvmeq;
1057
1058 free_cqdma:
68b8eca5 1059 dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
b60503ba
MW		 1060 nvmeq->cq_dma_addr);
1061 free_nvmeq:
1062 kfree(nvmeq);
1063 return NULL;
1064}
1065
3001082c
MW		 1066static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1067 const char *name)
1068{
58ffacb5
MW		 1069 if (use_threaded_interrupts)
1070 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
ec6ce618 1071 nvme_irq_check, nvme_irq,
58ffacb5
MW		 1072 IRQF_DISABLED | IRQF_SHARED,
1073 name, nvmeq);
3001082c
MW		 1074 return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1075 IRQF_DISABLED | IRQF_SHARED, name, nvmeq);
1076}
1077
8d85fce7
GKH		 1078static struct nvme_queue *nvme_create_queue(struct nvme_dev *dev, int qid,
1079 int cq_size, int vector)
b60503ba
MW		 1080{
1081 int result;
1082 struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector);
1083
3f85d50b 1084 if (!nvmeq)
6f0f5449 1085 return ERR_PTR(-ENOMEM);
3f85d50b 1086
b60503ba
MW		 1087 result = adapter_alloc_cq(dev, qid, nvmeq);
1088 if (result < 0)
1089 goto free_nvmeq;
1090
1091 result = adapter_alloc_sq(dev, qid, nvmeq);
1092 if (result < 0)
1093 goto release_cq;
1094
3001082c 1095 result = queue_request_irq(dev, nvmeq, "nvme");
b60503ba
MW		 1096 if (result < 0)
1097 goto release_sq;
1098
1099 return nvmeq;
1100
1101 release_sq:
1102 adapter_delete_sq(dev, qid);
1103 release_cq:
1104 adapter_delete_cq(dev, qid);
1105 free_nvmeq:
1106 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1107 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1108 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1109 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1110 kfree(nvmeq);
6f0f5449 1111 return ERR_PTR(result);
b60503ba
MW		 1112}
1113
ba47e386
MW		 1114static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
1115{
1116 unsigned long timeout;
1117 u32 bit = enabled ? NVME_CSTS_RDY : 0;
1118
1119 timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
1120
1121 while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
1122 msleep(100);
1123 if (fatal_signal_pending(current))
1124 return -EINTR;
1125 if (time_after(jiffies, timeout)) {
1126 dev_err(&dev->pci_dev->dev,
1127 "Device not ready; aborting initialisation\n");
1128 return -ENODEV;
1129 }
1130 }
1131
1132 return 0;
1133}
1134
1135/*
1136 * If the device has been passed off to us in an enabled state, just clear
1137 * the enabled bit. The spec says we should set the 'shutdown notification
1138 * bits', but doing so may cause the device to complete commands to the
1139 * admin queue ... and we don't know what memory that might be pointing at!
1140 */
1141static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
1142{
44af146a
MW		 1143 u32 cc = readl(&dev->bar->cc);
1144
1145 if (cc & NVME_CC_ENABLE)
1146 writel(cc & ~NVME_CC_ENABLE, &dev->bar->cc);
ba47e386
MW		 1147 return nvme_wait_ready(dev, cap, false);
1148}
1149
1150static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
1151{
1152 return nvme_wait_ready(dev, cap, true);
1153}
1154
8d85fce7 1155static int nvme_configure_admin_queue(struct nvme_dev *dev)
b60503ba 1156{
ba47e386 1157 int result;
b60503ba 1158 u32 aqa;
ba47e386 1159 u64 cap = readq(&dev->bar->cap);
b60503ba
MW		 1160 struct nvme_queue *nvmeq;
1161
1162 dev->dbs = ((void __iomem *)dev->bar) + 4096;
ba47e386
MW		 1163 dev->db_stride = NVME_CAP_STRIDE(cap);
1164
1165 result = nvme_disable_ctrl(dev, cap);
1166 if (result < 0)
1167 return result;
b60503ba
MW		 1168
1169 nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
3f85d50b
MW		 1170 if (!nvmeq)
1171 return -ENOMEM;
b60503ba
MW		 1172
1173 aqa = nvmeq->q_depth - 1;
1174 aqa |= aqa << 16;
1175
1176 dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
1177 dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1178 dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
7f53f9d2 1179 dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
b60503ba
MW		 1180
1181 writel(aqa, &dev->bar->aqa);
1182 writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
1183 writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
1184 writel(dev->ctrl_config, &dev->bar->cc);
1185
ba47e386 1186 result = nvme_enable_ctrl(dev, cap);
025c557a
KB		 1187 if (result)
1188 goto free_q;
9e866774 1189
3001082c 1190 result = queue_request_irq(dev, nvmeq, "nvme admin");
025c557a
KB		 1191 if (result)
1192 goto free_q;
1193
b60503ba
MW		 1194 dev->queues[0] = nvmeq;
1195 return result;
025c557a
KB		 1196
1197 free_q:
1198 nvme_free_queue_mem(nvmeq);
1199 return result;
b60503ba
MW		 1200}
1201
5d0f6131 1202struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
eca18b23 1203 unsigned long addr, unsigned length)
b60503ba 1204{
36c14ed9 1205 int i, err, count, nents, offset;
7fc3cdab
MW		 1206 struct scatterlist *sg;
1207 struct page **pages;
eca18b23 1208 struct nvme_iod *iod;
36c14ed9
MW		 1209
1210 if (addr & 3)
eca18b23 1211 return ERR_PTR(-EINVAL);
5460fc03 1212 if (!length || length > INT_MAX - PAGE_SIZE)
eca18b23 1213 return ERR_PTR(-EINVAL);
7fc3cdab 1214
36c14ed9 1215 offset = offset_in_page(addr);
7fc3cdab
MW		 1216 count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
1217 pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
22fff826
DC		 1218 if (!pages)
1219 return ERR_PTR(-ENOMEM);
36c14ed9
MW		 1220
1221 err = get_user_pages_fast(addr, count, 1, pages);
1222 if (err < count) {
1223 count = err;
1224 err = -EFAULT;
1225 goto put_pages;
1226 }
7fc3cdab 1227
eca18b23
MW		 1228 iod = nvme_alloc_iod(count, length, GFP_KERNEL);
1229 sg = iod->sg;
36c14ed9 1230 sg_init_table(sg, count);
d0ba1e49
MW		 1231 for (i = 0; i < count; i++) {
1232 sg_set_page(&sg[i], pages[i],
5460fc03
DC		 1233 min_t(unsigned, length, PAGE_SIZE - offset),
1234 offset);
d0ba1e49
MW		 1235 length -= (PAGE_SIZE - offset);
1236 offset = 0;
7fc3cdab 1237 }
fe304c43 1238 sg_mark_end(&sg[i - 1]);
1c2ad9fa 1239 iod->nents = count;
7fc3cdab
MW		 1240
1241 err = -ENOMEM;
1242 nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
1243 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
36c14ed9 1244 if (!nents)
eca18b23 1245 goto free_iod;
b60503ba 1246
7fc3cdab 1247 kfree(pages);
eca18b23 1248 return iod;
b60503ba 1249
eca18b23
MW		 1250 free_iod:
1251 kfree(iod);
7fc3cdab
MW		 1252 put_pages:
1253 for (i = 0; i < count; i++)
1254 put_page(pages[i]);
1255 kfree(pages);
eca18b23 1256 return ERR_PTR(err);
7fc3cdab 1257}
b60503ba 1258
5d0f6131 1259void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
1c2ad9fa 1260 struct nvme_iod *iod)
7fc3cdab 1261{
1c2ad9fa 1262 int i;
b60503ba 1263
1c2ad9fa
MW		 1264 dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
1265 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
7fc3cdab 1266
1c2ad9fa
MW		 1267 for (i = 0; i < iod->nents; i++)
1268 put_page(sg_page(&iod->sg[i]));
7fc3cdab 1269}
b60503ba 1270
a53295b6
MW		 1271static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1272{
1273 struct nvme_dev *dev = ns->dev;
1274 struct nvme_queue *nvmeq;
1275 struct nvme_user_io io;
1276 struct nvme_command c;
f410c680
KB		 1277 unsigned length, meta_len;
1278 int status, i;
1279 struct nvme_iod *iod, *meta_iod = NULL;
1280 dma_addr_t meta_dma_addr;
1281 void *meta, *uninitialized_var(meta_mem);
a53295b6
MW		 1282
1283 if (copy_from_user(&io, uio, sizeof(io)))
1284 return -EFAULT;
6c7d4945 1285 length = (io.nblocks + 1) << ns->lba_shift;
f410c680
KB		 1286 meta_len = (io.nblocks + 1) * ns->ms;
1287
1288 if (meta_len && ((io.metadata & 3) || !io.metadata))
1289 return -EINVAL;
6c7d4945
MW		 1290
1291 switch (io.opcode) {
1292 case nvme_cmd_write:
1293 case nvme_cmd_read:
6bbf1acd 1294 case nvme_cmd_compare:
eca18b23 1295 iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
6413214c 1296 break;
6c7d4945 1297 default:
6bbf1acd 1298 return -EINVAL;
6c7d4945
MW		 1299 }
1300
eca18b23
MW		 1301 if (IS_ERR(iod))
1302 return PTR_ERR(iod);
a53295b6
MW		 1303
1304 memset(&c, 0, sizeof(c));
1305 c.rw.opcode = io.opcode;
1306 c.rw.flags = io.flags;
6c7d4945 1307 c.rw.nsid = cpu_to_le32(ns->ns_id);
a53295b6 1308 c.rw.slba = cpu_to_le64(io.slba);
6c7d4945 1309 c.rw.length = cpu_to_le16(io.nblocks);
a53295b6 1310 c.rw.control = cpu_to_le16(io.control);
1c9b5265
MW		 1311 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1312 c.rw.reftag = cpu_to_le32(io.reftag);
1313 c.rw.apptag = cpu_to_le16(io.apptag);
1314 c.rw.appmask = cpu_to_le16(io.appmask);
f410c680
KB		 1315
1316 if (meta_len) {
1317 meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata, meta_len);
1318 if (IS_ERR(meta_iod)) {
1319 status = PTR_ERR(meta_iod);
1320 meta_iod = NULL;
1321 goto unmap;
1322 }
1323
1324 meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
1325 &meta_dma_addr, GFP_KERNEL);
1326 if (!meta_mem) {
1327 status = -ENOMEM;
1328 goto unmap;
1329 }
1330
1331 if (io.opcode & 1) {
1332 int meta_offset = 0;
1333
1334 for (i = 0; i < meta_iod->nents; i++) {
1335 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1336 meta_iod->sg[i].offset;
1337 memcpy(meta_mem + meta_offset, meta,
1338 meta_iod->sg[i].length);
1339 kunmap_atomic(meta);
1340 meta_offset += meta_iod->sg[i].length;
1341 }
1342 }
1343
1344 c.rw.metadata = cpu_to_le64(meta_dma_addr);
1345 }
1346
eca18b23 1347 length = nvme_setup_prps(dev, &c.common, iod, length, GFP_KERNEL);
a53295b6 1348
040a93b5 1349 nvmeq = get_nvmeq(dev);
fa922821
MW		 1350 /*
1351 * Since nvme_submit_sync_cmd sleeps, we can't keep preemption
b1ad37ef
MW		 1352 * disabled. We may be preempted at any point, and be rescheduled
1353 * to a different CPU. That will cause cacheline bouncing, but no
1354 * additional races since q_lock already protects against other CPUs.
1355 */
a53295b6 1356 put_nvmeq(nvmeq);
b77954cb
MW		 1357 if (length != (io.nblocks + 1) << ns->lba_shift)
1358 status = -ENOMEM;
1359 else
ff976d72 1360 status = nvme_submit_sync_cmd(nvmeq, &c, NULL, NVME_IO_TIMEOUT);
a53295b6 1361
f410c680
KB		 1362 if (meta_len) {
1363 if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
1364 int meta_offset = 0;
1365
1366 for (i = 0; i < meta_iod->nents; i++) {
1367 meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
1368 meta_iod->sg[i].offset;
1369 memcpy(meta, meta_mem + meta_offset,
1370 meta_iod->sg[i].length);
1371 kunmap_atomic(meta);
1372 meta_offset += meta_iod->sg[i].length;
1373 }
1374 }
1375
1376 dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
1377 meta_dma_addr);
1378 }
1379
1380 unmap:
1c2ad9fa 1381 nvme_unmap_user_pages(dev, io.opcode & 1, iod);
eca18b23 1382 nvme_free_iod(dev, iod);
f410c680
KB		 1383
1384 if (meta_iod) {
1385 nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
1386 nvme_free_iod(dev, meta_iod);
1387 }
1388
a53295b6
MW		 1389 return status;
1390}
1391
50af8bae 1392static int nvme_user_admin_cmd(struct nvme_dev *dev,
6bbf1acd 1393 struct nvme_admin_cmd __user *ucmd)
6ee44cdc 1394{
6bbf1acd 1395 struct nvme_admin_cmd cmd;
6ee44cdc 1396 struct nvme_command c;
eca18b23 1397 int status, length;
c7d36ab8 1398 struct nvme_iod *uninitialized_var(iod);
94f370ca 1399 unsigned timeout;
6ee44cdc 1400
6bbf1acd
MW		 1401 if (!capable(CAP_SYS_ADMIN))
1402 return -EACCES;
1403 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
6ee44cdc 1404 return -EFAULT;
6ee44cdc
MW		 1405
1406 memset(&c, 0, sizeof(c));
6bbf1acd
MW		 1407 c.common.opcode = cmd.opcode;
1408 c.common.flags = cmd.flags;
1409 c.common.nsid = cpu_to_le32(cmd.nsid);
1410 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1411 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1412 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1413 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1414 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1415 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1416 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1417 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
1418
1419 length = cmd.data_len;
1420 if (cmd.data_len) {
49742188
MW		 1421 iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
1422 length);
eca18b23
MW		 1423 if (IS_ERR(iod))
1424 return PTR_ERR(iod);
1425 length = nvme_setup_prps(dev, &c.common, iod, length,
1426 GFP_KERNEL);
6bbf1acd
MW		 1427 }
1428
94f370ca
KB		 1429 timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
1430 ADMIN_TIMEOUT;
6bbf1acd 1431 if (length != cmd.data_len)
b77954cb
MW		 1432 status = -ENOMEM;
1433 else
94f370ca
KB		 1434 status = nvme_submit_sync_cmd(dev->queues[0], &c, &cmd.result,
1435 timeout);
eca18b23 1436
6bbf1acd 1437 if (cmd.data_len) {
1c2ad9fa 1438 nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
eca18b23 1439 nvme_free_iod(dev, iod);
6bbf1acd 1440 }
f4f117f6 1441
cf90bc48 1442 if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
f4f117f6
KB		 1443 sizeof(cmd.result)))
1444 status = -EFAULT;
1445
6ee44cdc
MW		 1446 return status;
1447}
1448
b60503ba
MW		 1449static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
1450 unsigned long arg)
1451{
1452 struct nvme_ns *ns = bdev->bd_disk->private_data;
1453
1454 switch (cmd) {
6bbf1acd
MW		 1455 case NVME_IOCTL_ID:
1456 return ns->ns_id;
1457 case NVME_IOCTL_ADMIN_CMD:
50af8bae 1458 return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
a53295b6
MW		 1459 case NVME_IOCTL_SUBMIT_IO:
1460 return nvme_submit_io(ns, (void __user *)arg);
5d0f6131
VV		 1461 case SG_GET_VERSION_NUM:
1462 return nvme_sg_get_version_num((void __user *)arg);
1463 case SG_IO:
1464 return nvme_sg_io(ns, (void __user *)arg);
b60503ba
MW		 1465 default:
1466 return -ENOTTY;
1467 }
1468}
1469
1470static const struct block_device_operations nvme_fops = {
1471 .owner = THIS_MODULE,
1472 .ioctl = nvme_ioctl,
49481682 1473 .compat_ioctl = nvme_ioctl,
b60503ba
MW		 1474};
1475
1fa6aead
MW		 1476static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
1477{
1478 while (bio_list_peek(&nvmeq->sq_cong)) {
1479 struct bio *bio = bio_list_pop(&nvmeq->sq_cong);
1480 struct nvme_ns *ns = bio->bi_bdev->bd_disk->private_data;
427e9708
KB		 1481
1482 if (bio_list_empty(&nvmeq->sq_cong))
1483 remove_wait_queue(&nvmeq->sq_full,
1484 &nvmeq->sq_cong_wait);
1fa6aead 1485 if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
427e9708
KB		 1486 if (bio_list_empty(&nvmeq->sq_cong))
1487 add_wait_queue(&nvmeq->sq_full,
1488 &nvmeq->sq_cong_wait);
1fa6aead
MW		 1489 bio_list_add_head(&nvmeq->sq_cong, bio);
1490 break;
1491 }
1492 }
1493}
1494
1495static int nvme_kthread(void *data)
1496{
1497 struct nvme_dev *dev;
1498
1499 while (!kthread_should_stop()) {
564a232c 1500 set_current_state(TASK_INTERRUPTIBLE);
1fa6aead
MW		 1501 spin_lock(&dev_list_lock);
1502 list_for_each_entry(dev, &dev_list, node) {
1503 int i;
1504 for (i = 0; i < dev->queue_count; i++) {
1505 struct nvme_queue *nvmeq = dev->queues[i];
740216fc
MW		 1506 if (!nvmeq)
1507 continue;
1fa6aead
MW		 1508 spin_lock_irq(&nvmeq->q_lock);
1509 if (nvme_process_cq(nvmeq))
1510 printk("process_cq did something\n");
a09115b2 1511 nvme_cancel_ios(nvmeq, true);
1fa6aead
MW		 1512 nvme_resubmit_bios(nvmeq);
1513 spin_unlock_irq(&nvmeq->q_lock);
1514 }
1515 }
1516 spin_unlock(&dev_list_lock);
acb7aa0d 1517 schedule_timeout(round_jiffies_relative(HZ));
1fa6aead
MW		 1518 }
1519 return 0;
1520}
1521
5aff9382
MW		 1522static DEFINE_IDA(nvme_index_ida);
1523
1524static int nvme_get_ns_idx(void)
1525{
1526 int index, error;
1527
1528 do {
1529 if (!ida_pre_get(&nvme_index_ida, GFP_KERNEL))
1530 return -1;
1531
1532 spin_lock(&dev_list_lock);
1533 error = ida_get_new(&nvme_index_ida, &index);
1534 spin_unlock(&dev_list_lock);
1535 } while (error == -EAGAIN);
1536
1537 if (error)
1538 index = -1;
1539 return index;
1540}
1541
1542static void nvme_put_ns_idx(int index)
1543{
1544 spin_lock(&dev_list_lock);
1545 ida_remove(&nvme_index_ida, index);
1546 spin_unlock(&dev_list_lock);
1547}
1548
0e5e4f0e
KB		 1549static void nvme_config_discard(struct nvme_ns *ns)
1550{
1551 u32 logical_block_size = queue_logical_block_size(ns->queue);
1552 ns->queue->limits.discard_zeroes_data = 0;
1553 ns->queue->limits.discard_alignment = logical_block_size;
1554 ns->queue->limits.discard_granularity = logical_block_size;
1555 ns->queue->limits.max_discard_sectors = 0xffffffff;
1556 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1557}
1558
5aff9382 1559static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid,
b60503ba
MW		 1560 struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
1561{
1562 struct nvme_ns *ns;
1563 struct gendisk *disk;
1564 int lbaf;
1565
1566 if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
1567 return NULL;
1568
1569 ns = kzalloc(sizeof(*ns), GFP_KERNEL);
1570 if (!ns)
1571 return NULL;
1572 ns->queue = blk_alloc_queue(GFP_KERNEL);
1573 if (!ns->queue)
1574 goto out_free_ns;
4eeb9215
MW		 1575 ns->queue->queue_flags = QUEUE_FLAG_DEFAULT;
1576 queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
1577 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
b60503ba
MW		 1578 blk_queue_make_request(ns->queue, nvme_make_request);
1579 ns->dev = dev;
1580 ns->queue->queuedata = ns;
1581
1582 disk = alloc_disk(NVME_MINORS);
1583 if (!disk)
1584 goto out_free_queue;
5aff9382 1585 ns->ns_id = nsid;
b60503ba
MW		 1586 ns->disk = disk;
1587 lbaf = id->flbas & 0xf;
1588 ns->lba_shift = id->lbaf[lbaf].ds;
f410c680 1589 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
e9ef4636 1590 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
8fc23e03
KB		 1591 if (dev->max_hw_sectors)
1592 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
b60503ba
MW		 1593
1594 disk->major = nvme_major;
1595 disk->minors = NVME_MINORS;
5aff9382 1596 disk->first_minor = NVME_MINORS * nvme_get_ns_idx();
b60503ba
MW		 1597 disk->fops = &nvme_fops;
1598 disk->private_data = ns;
1599 disk->queue = ns->queue;
388f037f 1600 disk->driverfs_dev = &dev->pci_dev->dev;
5aff9382 1601 sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
b60503ba
MW		 1602 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1603
0e5e4f0e
KB		 1604 if (dev->oncs & NVME_CTRL_ONCS_DSM)
1605 nvme_config_discard(ns);
1606
b60503ba
MW		 1607 return ns;
1608
1609 out_free_queue:
1610 blk_cleanup_queue(ns->queue);
1611 out_free_ns:
1612 kfree(ns);
1613 return NULL;
1614}
1615
1616static void nvme_ns_free(struct nvme_ns *ns)
1617{
5aff9382 1618 int index = ns->disk->first_minor / NVME_MINORS;
b60503ba 1619 put_disk(ns->disk);
5aff9382 1620 nvme_put_ns_idx(index);
b60503ba
MW		 1621 blk_cleanup_queue(ns->queue);
1622 kfree(ns);
1623}
1624
b3b06812 1625static int set_queue_count(struct nvme_dev *dev, int count)
b60503ba
MW		 1626{
1627 int status;
1628 u32 result;
b3b06812 1629 u32 q_count = (count - 1) | ((count - 1) << 16);
b60503ba 1630
df348139 1631 status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
bc5fc7e4 1632 &result);
b60503ba
MW		 1633 if (status)
1634 return -EIO;
1635 return min(result & 0xffff, result >> 16) + 1;
1636}
1637
8d85fce7 1638static int nvme_setup_io_queues(struct nvme_dev *dev)
b60503ba 1639{
fa08a396
RRG		 1640 struct pci_dev *pdev = dev->pci_dev;
1641 int result, cpu, i, nr_io_queues, db_bar_size, q_depth, q_count;
b60503ba 1642
b348b7d5
MW		 1643 nr_io_queues = num_online_cpus();
1644 result = set_queue_count(dev, nr_io_queues);
1b23484b
MW		 1645 if (result < 0)
1646 return result;
b348b7d5
MW		 1647 if (result < nr_io_queues)
1648 nr_io_queues = result;
b60503ba 1649
fa08a396 1650 q_count = nr_io_queues;
1b23484b
MW		 1651 /* Deregister the admin queue's interrupt */
1652 free_irq(dev->entry[0].vector, dev->queues[0]);
1653
f1938f6e
MW		 1654 db_bar_size = 4096 + ((nr_io_queues + 1) << (dev->db_stride + 3));
1655 if (db_bar_size > 8192) {
1656 iounmap(dev->bar);
fa08a396 1657 dev->bar = ioremap(pci_resource_start(pdev, 0), db_bar_size);
f1938f6e
MW		 1658 dev->dbs = ((void __iomem *)dev->bar) + 4096;
1659 dev->queues[0]->q_db = dev->dbs;
1660 }
1661
b348b7d5 1662 for (i = 0; i < nr_io_queues; i++)
1b23484b
MW		 1663 dev->entry[i].entry = i;
1664 for (;;) {
fa08a396 1665 result = pci_enable_msix(pdev, dev->entry, nr_io_queues);
1b23484b
MW		 1666 if (result == 0) {
1667 break;
1668 } else if (result > 0) {
b348b7d5 1669 nr_io_queues = result;
1b23484b
MW		 1670 continue;
1671 } else {
fa08a396 1672 nr_io_queues = 0;
1b23484b
MW		 1673 break;
1674 }
1675 }
1676
fa08a396
RRG		 1677 if (nr_io_queues == 0) {
1678 nr_io_queues = q_count;
1679 for (;;) {
1680 result = pci_enable_msi_block(pdev, nr_io_queues);
1681 if (result == 0) {
1682 for (i = 0; i < nr_io_queues; i++)
1683 dev->entry[i].vector = i + pdev->irq;
1684 break;
1685 } else if (result > 0) {
1686 nr_io_queues = result;
1687 continue;
1688 } else {
1689 nr_io_queues = 1;
1690 break;
1691 }
1692 }
1693 }
1694
1b23484b
MW		 1695 result = queue_request_irq(dev, dev->queues[0], "nvme admin");
1696 /* XXX: handle failure here */
1697
1698 cpu = cpumask_first(cpu_online_mask);
b348b7d5 1699 for (i = 0; i < nr_io_queues; i++) {
1b23484b
MW		 1700 irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
1701 cpu = cpumask_next(cpu, cpu_online_mask);
1702 }
1703
a0cadb85
KB		 1704 q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
1705 NVME_Q_DEPTH);
b348b7d5 1706 for (i = 0; i < nr_io_queues; i++) {
a0cadb85 1707 dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i);
6f0f5449
MW		 1708 if (IS_ERR(dev->queues[i + 1]))
1709 return PTR_ERR(dev->queues[i + 1]);
1b23484b
MW		 1710 dev->queue_count++;
1711 }
b60503ba 1712
9ecdc946
MW		 1713 for (; i < num_possible_cpus(); i++) {
1714 int target = i % rounddown_pow_of_two(dev->queue_count - 1);
1715 dev->queues[i + 1] = dev->queues[target + 1];
1716 }
1717
b60503ba
MW		 1718 return 0;
1719}
1720
1721static void nvme_free_queues(struct nvme_dev *dev)
1722{
1723 int i;
1724
1725 for (i = dev->queue_count - 1; i >= 0; i--)
1726 nvme_free_queue(dev, i);
1727}
1728
422ef0c7
MW		 1729/*
1730 * Return: error value if an error occurred setting up the queues or calling
1731 * Identify Device. 0 if these succeeded, even if adding some of the
1732 * namespaces failed. At the moment, these failures are silent. TBD which
1733 * failures should be reported.
1734 */
8d85fce7 1735static int nvme_dev_add(struct nvme_dev *dev)
b60503ba
MW		 1736{
1737 int res, nn, i;
cbb6218f 1738 struct nvme_ns *ns;
51814232 1739 struct nvme_id_ctrl *ctrl;
bc5fc7e4
MW		 1740 struct nvme_id_ns *id_ns;
1741 void *mem;
b60503ba 1742 dma_addr_t dma_addr;
159b67d7 1743 int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
b60503ba
MW		 1744
1745 res = nvme_setup_io_queues(dev);
1746 if (res)
1747 return res;
1748
bc5fc7e4 1749 mem = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
b60503ba 1750 GFP_KERNEL);
a9ef4343
KB		 1751 if (!mem)
1752 return -ENOMEM;
b60503ba 1753
bc5fc7e4 1754 res = nvme_identify(dev, 0, 1, dma_addr);
b60503ba
MW		 1755 if (res) {
1756 res = -EIO;
cbb6218f 1757 goto out;
b60503ba
MW		 1758 }
1759
bc5fc7e4 1760 ctrl = mem;
51814232 1761 nn = le32_to_cpup(&ctrl->nn);
0e5e4f0e 1762 dev->oncs = le16_to_cpup(&ctrl->oncs);
51814232
MW		 1763 memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
1764 memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
1765 memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
159b67d7 1766 if (ctrl->mdts)
8fc23e03 1767 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
159b67d7
KB		 1768 if ((dev->pci_dev->vendor == PCI_VENDOR_ID_INTEL) &&
1769 (dev->pci_dev->device == 0x0953) && ctrl->vs[3])
1770 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
b60503ba 1771
bc5fc7e4 1772 id_ns = mem;
2b2c1896 1773 for (i = 1; i <= nn; i++) {
bc5fc7e4 1774 res = nvme_identify(dev, i, 0, dma_addr);
b60503ba
MW		 1775 if (res)
1776 continue;
1777
bc5fc7e4 1778 if (id_ns->ncap == 0)
b60503ba
MW		 1779 continue;
1780
bc5fc7e4 1781 res = nvme_get_features(dev, NVME_FEAT_LBA_RANGE, i,
08df1e05 1782 dma_addr + 4096, NULL);
b60503ba 1783 if (res)
12209036 1784 memset(mem + 4096, 0, 4096);
b60503ba 1785
bc5fc7e4 1786 ns = nvme_alloc_ns(dev, i, mem, mem + 4096);
b60503ba
MW		 1787 if (ns)
1788 list_add_tail(&ns->list, &dev->namespaces);
1789 }
1790 list_for_each_entry(ns, &dev->namespaces, list)
1791 add_disk(ns->disk);
422ef0c7 1792 res = 0;
b60503ba 1793
bc5fc7e4 1794 out:
684f5c20 1795 dma_free_coherent(&dev->pci_dev->dev, 8192, mem, dma_addr);
b60503ba
MW		 1796 return res;
1797}
1798
1799static int nvme_dev_remove(struct nvme_dev *dev)
1800{
1801 struct nvme_ns *ns, *next;
1802
1fa6aead
MW		 1803 spin_lock(&dev_list_lock);
1804 list_del(&dev->node);
1805 spin_unlock(&dev_list_lock);
1806
b60503ba
MW		 1807 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1808 list_del(&ns->list);
1809 del_gendisk(ns->disk);
1810 nvme_ns_free(ns);
1811 }
1812
1813 nvme_free_queues(dev);
1814
1815 return 0;
1816}
1817
091b6092
MW		 1818static int nvme_setup_prp_pools(struct nvme_dev *dev)
1819{
1820 struct device *dmadev = &dev->pci_dev->dev;
1821 dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
1822 PAGE_SIZE, PAGE_SIZE, 0);
1823 if (!dev->prp_page_pool)
1824 return -ENOMEM;
1825
99802a7a
MW		 1826 /* Optimisation for I/Os between 4k and 128k */
1827 dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
1828 256, 256, 0);
1829 if (!dev->prp_small_pool) {
1830 dma_pool_destroy(dev->prp_page_pool);
1831 return -ENOMEM;
1832 }
091b6092
MW		 1833 return 0;
1834}
1835
1836static void nvme_release_prp_pools(struct nvme_dev *dev)
1837{
1838 dma_pool_destroy(dev->prp_page_pool);
99802a7a 1839 dma_pool_destroy(dev->prp_small_pool);
091b6092
MW		 1840}
1841
cd58ad7d
QSA		 1842static DEFINE_IDA(nvme_instance_ida);
1843
1844static int nvme_set_instance(struct nvme_dev *dev)
b60503ba 1845{
cd58ad7d
QSA		 1846 int instance, error;
1847
1848 do {
1849 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
1850 return -ENODEV;
1851
1852 spin_lock(&dev_list_lock);
1853 error = ida_get_new(&nvme_instance_ida, &instance);
1854 spin_unlock(&dev_list_lock);
1855 } while (error == -EAGAIN);
1856
1857 if (error)
1858 return -ENODEV;
1859
1860 dev->instance = instance;
1861 return 0;
b60503ba
MW		 1862}
1863
1864static void nvme_release_instance(struct nvme_dev *dev)
1865{
cd58ad7d
QSA		 1866 spin_lock(&dev_list_lock);
1867 ida_remove(&nvme_instance_ida, dev->instance);
1868 spin_unlock(&dev_list_lock);
b60503ba
MW		 1869}
1870
5e82e952
KB		 1871static void nvme_free_dev(struct kref *kref)
1872{
1873 struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
1874 nvme_dev_remove(dev);
fa08a396
RRG		 1875 if (dev->pci_dev->msi_enabled)
1876 pci_disable_msi(dev->pci_dev);
1877 else if (dev->pci_dev->msix_enabled)
1878 pci_disable_msix(dev->pci_dev);
5e82e952
KB		 1879 iounmap(dev->bar);
1880 nvme_release_instance(dev);
1881 nvme_release_prp_pools(dev);
1882 pci_disable_device(dev->pci_dev);
1883 pci_release_regions(dev->pci_dev);
1884 kfree(dev->queues);
1885 kfree(dev->entry);
1886 kfree(dev);
1887}
1888
1889static int nvme_dev_open(struct inode *inode, struct file *f)
1890{
1891 struct nvme_dev *dev = container_of(f->private_data, struct nvme_dev,
1892 miscdev);
1893 kref_get(&dev->kref);
1894 f->private_data = dev;
1895 return 0;
1896}
1897
1898static int nvme_dev_release(struct inode *inode, struct file *f)
1899{
1900 struct nvme_dev *dev = f->private_data;
1901 kref_put(&dev->kref, nvme_free_dev);
1902 return 0;
1903}
1904
1905static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1906{
1907 struct nvme_dev *dev = f->private_data;
1908 switch (cmd) {
1909 case NVME_IOCTL_ADMIN_CMD:
1910 return nvme_user_admin_cmd(dev, (void __user *)arg);
1911 default:
1912 return -ENOTTY;
1913 }
1914}
1915
1916static const struct file_operations nvme_dev_fops = {
1917 .owner = THIS_MODULE,
1918 .open = nvme_dev_open,
1919 .release = nvme_dev_release,
1920 .unlocked_ioctl = nvme_dev_ioctl,
1921 .compat_ioctl = nvme_dev_ioctl,
1922};
1923
8d85fce7 1924static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
b60503ba 1925{
574e8b95 1926 int bars, result = -ENOMEM;
b60503ba
MW		 1927 struct nvme_dev *dev;
1928
1929 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1930 if (!dev)
1931 return -ENOMEM;
1932 dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
1933 GFP_KERNEL);
1934 if (!dev->entry)
1935 goto free;
1b23484b
MW		 1936 dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
1937 GFP_KERNEL);
b60503ba
MW		 1938 if (!dev->queues)
1939 goto free;
1940
0ee5a7d7
SMM		 1941 if (pci_enable_device_mem(pdev))
1942 goto free;
f64d3365 1943 pci_set_master(pdev);
574e8b95
MW		 1944 bars = pci_select_bars(pdev, IORESOURCE_MEM);
1945 if (pci_request_selected_regions(pdev, bars, "nvme"))
1946 goto disable;
0ee5a7d7 1947
b60503ba
MW		 1948 INIT_LIST_HEAD(&dev->namespaces);
1949 dev->pci_dev = pdev;
1950 pci_set_drvdata(pdev, dev);
cf9f123b
MW		 1951
1952 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)))
1953 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1954 else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)))
1955 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1956 else
1957 goto disable;
1958
cd58ad7d
QSA		 1959 result = nvme_set_instance(dev);
1960 if (result)
1961 goto disable;
1962
53c9577e 1963 dev->entry[0].vector = pdev->irq;
b60503ba 1964
091b6092
MW		 1965 result = nvme_setup_prp_pools(dev);
1966 if (result)
1967 goto disable_msix;
1968
b60503ba
MW		 1969 dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1970 if (!dev->bar) {
1971 result = -ENOMEM;
574e8b95 1972 goto disable_msix;
b60503ba
MW		 1973 }
1974
1975 result = nvme_configure_admin_queue(dev);
1976 if (result)
1977 goto unmap;
1978 dev->queue_count++;
1979
1fa6aead
MW		 1980 spin_lock(&dev_list_lock);
1981 list_add(&dev->node, &dev_list);
1982 spin_unlock(&dev_list_lock);
1983
740216fc
MW		 1984 result = nvme_dev_add(dev);
1985 if (result)
1986 goto delete;
1987
5e82e952
KB		 1988 scnprintf(dev->name, sizeof(dev->name), "nvme%d", dev->instance);
1989 dev->miscdev.minor = MISC_DYNAMIC_MINOR;
1990 dev->miscdev.parent = &pdev->dev;
1991 dev->miscdev.name = dev->name;
1992 dev->miscdev.fops = &nvme_dev_fops;
1993 result = misc_register(&dev->miscdev);
1994 if (result)
1995 goto remove;
1996
1997 kref_init(&dev->kref);
b60503ba
MW		 1998 return 0;
1999
5e82e952
KB		 2000 remove:
2001 nvme_dev_remove(dev);
b60503ba 2002 delete:
740216fc
MW		 2003 spin_lock(&dev_list_lock);
2004 list_del(&dev->node);
2005 spin_unlock(&dev_list_lock);
2006
b60503ba
MW		 2007 nvme_free_queues(dev);
2008 unmap:
2009 iounmap(dev->bar);
574e8b95 2010 disable_msix:
fa08a396
RRG		 2011 if (dev->pci_dev->msi_enabled)
2012 pci_disable_msi(dev->pci_dev);
2013 else if (dev->pci_dev->msix_enabled)
2014 pci_disable_msix(dev->pci_dev);
b60503ba 2015 nvme_release_instance(dev);
091b6092 2016 nvme_release_prp_pools(dev);
574e8b95 2017 disable:
0ee5a7d7 2018 pci_disable_device(pdev);
574e8b95 2019 pci_release_regions(pdev);
b60503ba
MW		 2020 free:
2021 kfree(dev->queues);
2022 kfree(dev->entry);
2023 kfree(dev);
2024 return result;
2025}
2026
8d85fce7 2027static void nvme_remove(struct pci_dev *pdev)
b60503ba
MW		 2028{
2029 struct nvme_dev *dev = pci_get_drvdata(pdev);
5e82e952
KB		 2030 misc_deregister(&dev->miscdev);
2031 kref_put(&dev->kref, nvme_free_dev);
b60503ba
MW		 2032}
2033
2034/* These functions are yet to be implemented */
2035#define nvme_error_detected NULL
2036#define nvme_dump_registers NULL
2037#define nvme_link_reset NULL
2038#define nvme_slot_reset NULL
2039#define nvme_error_resume NULL
2040#define nvme_suspend NULL
2041#define nvme_resume NULL
2042
1d352035 2043static const struct pci_error_handlers nvme_err_handler = {
b60503ba
MW		 2044 .error_detected = nvme_error_detected,
2045 .mmio_enabled = nvme_dump_registers,
2046 .link_reset = nvme_link_reset,
2047 .slot_reset = nvme_slot_reset,
2048 .resume = nvme_error_resume,
2049};
2050
2051/* Move to pci_ids.h later */
2052#define PCI_CLASS_STORAGE_EXPRESS 0x010802
2053
2054static DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
2055 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2056 { 0, }
2057};
2058MODULE_DEVICE_TABLE(pci, nvme_id_table);
2059
2060static struct pci_driver nvme_driver = {
2061 .name = "nvme",
2062 .id_table = nvme_id_table,
2063 .probe = nvme_probe,
8d85fce7 2064 .remove = nvme_remove,
b60503ba
MW		 2065 .suspend = nvme_suspend,
2066 .resume = nvme_resume,
2067 .err_handler = &nvme_err_handler,
2068};
2069
2070static int __init nvme_init(void)
2071{
0ac13140 2072 int result;
1fa6aead
MW		 2073
2074 nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2075 if (IS_ERR(nvme_thread))
2076 return PTR_ERR(nvme_thread);
b60503ba 2077
5c42ea16
KB		 2078 result = register_blkdev(nvme_major, "nvme");
2079 if (result < 0)
1fa6aead 2080 goto kill_kthread;
5c42ea16 2081 else if (result > 0)
0ac13140 2082 nvme_major = result;
b60503ba
MW		 2083
2084 result = pci_register_driver(&nvme_driver);
1fa6aead
MW		 2085 if (result)
2086 goto unregister_blkdev;
2087 return 0;
b60503ba 2088
1fa6aead 2089 unregister_blkdev:
b60503ba 2090 unregister_blkdev(nvme_major, "nvme");
1fa6aead
MW		 2091 kill_kthread:
2092 kthread_stop(nvme_thread);
b60503ba
MW		 2093 return result;
2094}
2095
2096static void __exit nvme_exit(void)
2097{
2098 pci_unregister_driver(&nvme_driver);
2099 unregister_blkdev(nvme_major, "nvme");
1fa6aead 2100 kthread_stop(nvme_thread);
b60503ba
MW		 2101}
2102
2103MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2104MODULE_LICENSE("GPL");
366e8217 2105MODULE_VERSION("0.8");
b60503ba
MW		 2106module_init(nvme_init);
2107module_exit(nvme_exit);
kernel source for telekom puls (alcatel/mediatek)