IB/rdmavt, hfi1: Fix NFSoRDMA failure with FRMR enabled

Hanging has been observed while writing a file over NFSoRDMA. Dmesg on
the server contains messages like these:

[  931.992501] svcrdma: Error -22 posting RDMA_READ
[  952.076879] svcrdma: Error -22 posting RDMA_READ
[  982.154127] svcrdma: Error -22 posting RDMA_READ
[ 1012.235884] svcrdma: Error -22 posting RDMA_READ
[ 1042.319194] svcrdma: Error -22 posting RDMA_READ

Here is why:

With the base memory management extension enabled, FRMR is used instead
of FMR. The xprtrdma server issues each RDMA read request as the following
bundle:

(1)IB_WR_REG_MR, signaled;
(2)IB_WR_RDMA_READ, signaled;
(3)IB_WR_LOCAL_INV, signaled & fencing.

These requests are signaled. In order to generate completion, the fast
register work request is processed by the hfi1 send engine after being
posted to the work queue, and the corresponding lkey is not valid until
the request is processed. However, the rdmavt driver validates lkey when
the RDMA read request is posted and thus it fails immediately with error
-EINVAL (-22).

This patch changes the work flow of local operations (fast register and
local invalidate) so that fast register work requests are always
processed immediately to ensure that the corresponding lkey is valid
when subsequent work requests are posted. Local invalidate requests are
processed immediately if fencing is not required and no previous local
invalidate request is pending.

To allow completion generation for signaled local operations that have
been processed before posting to the work queue, an internal send flag
RVT_SEND_COMPLETION_ONLY is added. The hfi1 send engine checks this flag
and only generates completion for such requests.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Jianxin Xiong <jianxin.xiong@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index 0bc43b67d0b8a1641b00d27af7b1fd0f5033e49f..5da190e6011b1390f7861b2bd89a5a09e7976c14 100644 (file)
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -402,7 +402,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        char newreq;
        int middle = 0;
        int delta;
-       int err;
 
        ps->s_txreq = get_txreq(ps->dev, qp);
        if (IS_ERR(ps->s_txreq))
@@ -484,25 +483,27 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
                         */
                        if (wqe->wr.opcode == IB_WR_REG_MR ||
                            wqe->wr.opcode == IB_WR_LOCAL_INV) {
+                               int local_ops = 0;
+                               int err = 0;
+
                                if (qp->s_last != qp->s_cur)
                                        goto bail;
                                if (++qp->s_cur == qp->s_size)
                                        qp->s_cur = 0;
                                if (++qp->s_tail == qp->s_size)
                                        qp->s_tail = 0;
-                               if (wqe->wr.opcode == IB_WR_REG_MR)
-                                       err = rvt_fast_reg_mr(
-                                               qp, wqe->reg_wr.mr,
-                                               wqe->reg_wr.key,
-                                               wqe->reg_wr.access);
-                               else
+                               if (!(wqe->wr.send_flags &
+                                     RVT_SEND_COMPLETION_ONLY)) {
                                        err = rvt_invalidate_rkey(
                                                qp,
                                                wqe->wr.ex.invalidate_rkey);
+                                       local_ops = 1;
+                               }
                                hfi1_send_complete(qp, wqe,
                                                   err ? IB_WC_LOC_PROT_ERR
                                                       : IB_WC_SUCCESS);
-                               atomic_dec(&qp->local_ops_pending);
+                               if (local_ops)
+                                       atomic_dec(&qp->local_ops_pending);
                                qp->s_hdrwords = 0;
                                goto done_free_tx;
                        }

diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index 76b9c9e42d86eff6db037ed3381e10343a4496bc..7e76d33a577410094e3cf0077dd8d72bed48f5ed 100644 (file)
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -442,16 +442,15 @@ again:
        sqp->s_len = wqe->length;
        switch (wqe->wr.opcode) {
        case IB_WR_REG_MR:
-               if (rvt_fast_reg_mr(sqp, wqe->reg_wr.mr, wqe->reg_wr.key,
-                                   wqe->reg_wr.access))
-                       send_status = IB_WC_LOC_PROT_ERR;
-               local_ops = 1;
                goto send_comp;
 
        case IB_WR_LOCAL_INV:
-               if (rvt_invalidate_rkey(sqp, wqe->wr.ex.invalidate_rkey))
-                       send_status = IB_WC_LOC_PROT_ERR;
-               local_ops = 1;
+               if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
+                       if (rvt_invalidate_rkey(sqp,
+                                               wqe->wr.ex.invalidate_rkey))
+                               send_status = IB_WC_LOC_PROT_ERR;
+                       local_ops = 1;
+               }
                goto send_comp;
 
        case IB_WR_SEND_WITH_INV:

diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index ef6c96cd3d68da35e3dcecb840289741159c129a..a726d96d185f3261ac77f99b73fe6112aa90c914 100644 (file)
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -77,7 +77,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
        u32 len;
        u32 pmtu = qp->pmtu;
        int middle = 0;
-       int err;
 
        ps->s_txreq = get_txreq(ps->dev, qp);
        if (IS_ERR(ps->s_txreq))
@@ -125,20 +124,22 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
                 */
                if (wqe->wr.opcode == IB_WR_REG_MR ||
                    wqe->wr.opcode == IB_WR_LOCAL_INV) {
+                       int local_ops = 0;
+                       int err = 0;
+
                        if (qp->s_last != qp->s_cur)
                                goto bail;
                        if (++qp->s_cur == qp->s_size)
                                qp->s_cur = 0;
-                       if (wqe->wr.opcode == IB_WR_REG_MR)
-                               err = rvt_fast_reg_mr(qp, wqe->reg_wr.mr,
-                                                     wqe->reg_wr.key,
-                                                     wqe->reg_wr.access);
-                       else
+                       if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {
                                err = rvt_invalidate_rkey(
                                        qp, wqe->wr.ex.invalidate_rkey);
+                               local_ops = 1;
+                       }
                        hfi1_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
                                                        : IB_WC_SUCCESS);
-                       atomic_dec(&qp->local_ops_pending);
+                       if (local_ops)
+                               atomic_dec(&qp->local_ops_pending);
                        qp->s_hdrwords = 0;
                        goto done_free_tx;
                }

diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 218494c6afe237cf29353024c7fe7a455f801ec1..8ccf1b970b2c3dde1d69c26b39837403e43062cb 100644 (file)
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -1579,6 +1579,7 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
        int ret;
        size_t cplen;
        bool reserved_op;
+       int local_ops_delayed = 0;
 
        BUILD_BUG_ON(IB_QPT_MAX >= (sizeof(u32) * BITS_PER_BYTE));
 
@@ -1592,25 +1593,37 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
        cplen = ret;
 
        /*
-        * Local operations including fast register and local invalidate
-        * can be processed immediately w/o being posted to the send queue
-        * if neither fencing nor completion generation is needed. However,
-        * once fencing or completion is requested, direct processing of
-        * following local operations must be disabled until all the local
-        * operations posted to the send queue have completed. This is
-        * necessary to ensure the correct ordering.
+        * Local operations include fast register and local invalidate.
+        * Fast register needs to be processed immediately because the
+        * registered lkey may be used by following work requests and the
+        * lkey needs to be valid at the time those requests are posted.
+        * Local invalidate can be processed immediately if fencing is
+        * not required and no previous local invalidate ops are pending.
+        * Signaled local operations that have been processed immediately
+        * need to have requests with "completion only" flags set posted
+        * to the send queue in order to generate completions.
         */
-       if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) &&
-           !(wr->send_flags & (IB_SEND_FENCE | IB_SEND_SIGNALED)) &&
-           !atomic_read(&qp->local_ops_pending)) {
-               struct ib_reg_wr *reg = reg_wr(wr);
-
+       if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) {
                switch (wr->opcode) {
                case IB_WR_REG_MR:
-                       return rvt_fast_reg_mr(qp, reg->mr, reg->key,
-                                              reg->access);
+                       ret = rvt_fast_reg_mr(qp,
+                                             reg_wr(wr)->mr,
+                                             reg_wr(wr)->key,
+                                             reg_wr(wr)->access);
+                       if (ret || !(wr->send_flags & IB_SEND_SIGNALED))
+                               return ret;
+                       break;
                case IB_WR_LOCAL_INV:
-                       return rvt_invalidate_rkey(qp, wr->ex.invalidate_rkey);
+                       if ((wr->send_flags & IB_SEND_FENCE) ||
+                           atomic_read(&qp->local_ops_pending)) {
+                               local_ops_delayed = 1;
+                       } else {
+                               ret = rvt_invalidate_rkey(
+                                       qp, wr->ex.invalidate_rkey);
+                               if (ret || !(wr->send_flags & IB_SEND_SIGNALED))
+                                       return ret;
+                       }
+                       break;
                default:
                        return -EINVAL;
                }
@@ -1675,7 +1688,10 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
        }
 
        if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) {
-               atomic_inc(&qp->local_ops_pending);
+               if (local_ops_delayed)
+                       atomic_inc(&qp->local_ops_pending);
+               else
+                       wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY;
                wqe->ssn = 0;
                wqe->psn = 0;
                wqe->lpsn = 0;

diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index 56adcfcabe0b09bfba2f79f3c862e720a13768f4..13902dd319a9a429196c1b53b5a1de448f041afa 100644 (file)
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -148,6 +148,7 @@
  * Internal send flags
  */
 #define RVT_SEND_RESERVE_USED           IB_SEND_RESERVED_START
+#define RVT_SEND_COMPLETION_ONLY       (IB_SEND_RESERVED_START << 1)
 
 /*
  * Send work request queue entry.