nvme: simplify completion handling

Now that all commands are executed as block layer requests we can remove the
internal completion in the NVMe driver.  Note that we can simply call
blk_mq_complete_request to abort commands as the block layer will protect
against double copletions internally.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Keith Busch <keith.busch@intel.com>
Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0497ff67324c39c43c054dc5b701aed8e07a3b02..84ac46fc98733e34944e04cab40a733698f7895c 100644 (file)
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -199,15 +199,11 @@ static inline void _nvme_check_size(void)
        BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 }
 
-typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
-                                               struct nvme_completion *);
-
 struct nvme_cmd_info {
-       nvme_completion_fn fn;
-       void *ctx;
        int aborted;
        struct nvme_queue *nvmeq;
-       struct nvme_iod iod[0];
+       struct nvme_iod *iod;
+       struct nvme_iod __iod;
 };
 
 /*
@@ -302,15 +298,6 @@ static int nvme_init_request(void *data, struct request *req,
        return 0;
 }
 
-static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
-                               nvme_completion_fn handler)
-{
-       cmd->fn = handler;
-       cmd->ctx = ctx;
-       cmd->aborted = 0;
-       blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
-}
-
 static void *iod_get_private(struct nvme_iod *iod)
 {
        return (void *) (iod->private & ~0x1UL);
@@ -324,44 +311,6 @@ static bool iod_should_kfree(struct nvme_iod *iod)
        return (iod->private & NVME_INT_MASK) == 0;
 }
 
-/* Special values must be less than 0x1000 */
-#define CMD_CTX_BASE           ((void *)POISON_POINTER_DELTA)
-#define CMD_CTX_CANCELLED      (0x30C + CMD_CTX_BASE)
-#define CMD_CTX_COMPLETED      (0x310 + CMD_CTX_BASE)
-#define CMD_CTX_INVALID                (0x314 + CMD_CTX_BASE)
-
-static void special_completion(struct nvme_queue *nvmeq, void *ctx,
-                                               struct nvme_completion *cqe)
-{
-       if (ctx == CMD_CTX_CANCELLED)
-               return;
-       if (ctx == CMD_CTX_COMPLETED) {
-               dev_warn(nvmeq->q_dmadev,
-                               "completed id %d twice on queue %d\n",
-                               cqe->command_id, le16_to_cpup(&cqe->sq_id));
-               return;
-       }
-       if (ctx == CMD_CTX_INVALID) {
-               dev_warn(nvmeq->q_dmadev,
-                               "invalid id %d completed on queue %d\n",
-                               cqe->command_id, le16_to_cpup(&cqe->sq_id));
-               return;
-       }
-       dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
-}
-
-static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
-{
-       void *ctx;
-
-       if (fn)
-               *fn = cmd->fn;
-       ctx = cmd->ctx;
-       cmd->fn = special_completion;
-       cmd->ctx = CMD_CTX_CANCELLED;
-       return ctx;
-}
-
 static void nvme_complete_async_event(struct nvme_dev *dev,
                struct nvme_completion *cqe)
 {
@@ -382,34 +331,6 @@ static void nvme_complete_async_event(struct nvme_dev *dev,
        }
 }
 
-static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
-                                 unsigned int tag)
-{
-       struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
-
-       return blk_mq_rq_to_pdu(req);
-}
-
-/*
- * Called with local interrupts disabled and the q_lock held.  May not sleep.
- */
-static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
-                                               nvme_completion_fn *fn)
-{
-       struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
-       void *ctx;
-       if (tag >= nvmeq->q_depth) {
-               *fn = special_completion;
-               return CMD_CTX_INVALID;
-       }
-       if (fn)
-               *fn = cmd->fn;
-       ctx = cmd->ctx;
-       cmd->fn = special_completion;
-       cmd->ctx = CMD_CTX_COMPLETED;
-       return ctx;
-}
-
 /**
  * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
  * @nvmeq: The queue to use
@@ -473,7 +394,7 @@ static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
            size <= NVME_INT_BYTES(dev)) {
                struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
 
-               iod = cmd->iod;
+               iod = &cmd->__iod;
                iod_init(iod, size, rq->nr_phys_segments,
                                (unsigned long) rq | NVME_INT_MASK);
                return iod;
@@ -570,12 +491,11 @@ static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 }
 #endif
 
-static void req_completion(struct nvme_queue *nvmeq, void *ctx,
-                                               struct nvme_completion *cqe)
+static void req_completion(struct nvme_queue *nvmeq, struct nvme_completion *cqe)
 {
-       struct nvme_iod *iod = ctx;
-       struct request *req = iod_get_private(iod);
+       struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
        struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
+       struct nvme_iod *iod = cmd_rq->iod;
        u16 status = le16_to_cpup(&cqe->status) >> 1;
        int error = 0;
 
@@ -586,14 +506,10 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx,
                        return;
                }
 
-               if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
-                       if (cmd_rq->ctx == CMD_CTX_CANCELLED)
-                               error = NVME_SC_CANCELLED;
-                       else
-                               error = status;
-               } else {
+               if (req->cmd_type == REQ_TYPE_DRV_PRIV)
+                       error = status;
+               else
                        error = nvme_error_status(status);
-               }
        }
 
        if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
@@ -836,8 +752,10 @@ static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
        if (ret)
                goto out;
 
+       cmd->iod = iod;
+       cmd->aborted = 0;
        cmnd.common.command_id = req->tag;
-       nvme_set_info(cmd, iod, req_completion);
+       blk_mq_start_request(req);
 
        spin_lock_irq(&nvmeq->q_lock);
        __nvme_submit_cmd(nvmeq, &cmnd);
@@ -857,8 +775,6 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
        phase = nvmeq->cq_phase;
 
        for (;;) {
-               void *ctx;
-               nvme_completion_fn fn;
                struct nvme_completion cqe = nvmeq->cqes[head];
                u16 status = le16_to_cpu(cqe.status);
 
@@ -873,6 +789,13 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
                if (tag && *tag == cqe.command_id)
                        *tag = -1;
 
+               if (unlikely(cqe.command_id >= nvmeq->q_depth)) {
+                       dev_warn(nvmeq->q_dmadev,
+                               "invalid id %d completed on queue %d\n",
+                               cqe.command_id, le16_to_cpu(cqe.sq_id));
+                       continue;
+               }
+
                /*
                 * AEN requests are special as they don't time out and can
                 * survive any kind of queue freeze and often don't respond to
@@ -885,8 +808,7 @@ static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
                        continue;
                }
 
-               ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
-               fn(nvmeq, ctx, &cqe);
+               req_completion(nvmeq, &cqe);
        }
 
        /* If the controller ignores the cq head doorbell and continuously
@@ -1125,29 +1047,18 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
 {
        struct nvme_queue *nvmeq = data;
-       void *ctx;
-       nvme_completion_fn fn;
-       struct nvme_cmd_info *cmd;
-       struct nvme_completion cqe;
+       int status;
 
        if (!blk_mq_request_started(req))
                return;
 
-       cmd = blk_mq_rq_to_pdu(req);
-
-       if (cmd->ctx == CMD_CTX_CANCELLED)
-               return;
+       dev_warn(nvmeq->q_dmadev,
+                "Cancelling I/O %d QID %d\n", req->tag, nvmeq->qid);
 
+       status = NVME_SC_CANCELLED;
        if (blk_queue_dying(req->q))
-               cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
-       else
-               cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
-
-
-       dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
-                                               req->tag, nvmeq->qid);
-       ctx = cancel_cmd_info(cmd, &fn);
-       fn(nvmeq, ctx, &cqe);
+               status |= NVME_SC_DNR;
+       blk_mq_complete_request(req, status);
 }
 
 static void nvme_free_queue(struct nvme_queue *nvmeq)