xprtrdma: Pre-mark remotely invalidated MRs

There are rare cases where an rpcrdma_req and its matched
rpcrdma_rep can be re-used, via rpcrdma_buffer_put, while the RPC
reply handler is still using that req. This is typically due to a
signal firing at just the wrong instant.

As part of closing this race window, avoid using the wrong
rpcrdma_rep to detect remotely invalidated MRs. Mark MRs as
invalidated while we are sure the rep is still OK to use.

BugLink: https://bugzilla.linux-nfs.org/show_bug.cgi?id=305
Fixes: 68791649a725 ('xprtrdma: Invalidate in the RPC reply ... ')
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index f81dd93176c0767cd783328710a042ffe22aff6d..31290cbedfe068a62da9ee5bcc4ab0e40fd9d14c 100644 (file)
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -464,7 +464,6 @@ static void
 frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
 {
        struct ib_send_wr *first, **prev, *last, *bad_wr;
-       struct rpcrdma_rep *rep = req->rl_reply;
        struct rpcrdma_ia *ia = &r_xprt->rx_ia;
        struct rpcrdma_frmr *f;
        struct rpcrdma_mw *mw;
@@ -483,8 +482,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
        list_for_each_entry(mw, &req->rl_registered, mw_list) {
                mw->frmr.fr_state = FRMR_IS_INVALID;
 
-               if ((rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) &&
-                   (mw->mw_handle == rep->rr_inv_rkey))
+               if (mw->mw_flags & RPCRDMA_MW_F_RI)
                        continue;
 
                f = &mw->frmr;

diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 694e9b13ecf07722848d86345589ad4793474fb5..2356a6305f25278250c0a8b1bad2405fbd1e76ec 100644 (file)
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -928,6 +928,24 @@ rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad)
        return fixup_copy_count;
 }
 
+/* Caller must guarantee @rep remains stable during this call.
+ */
+static void
+rpcrdma_mark_remote_invalidation(struct list_head *mws,
+                                struct rpcrdma_rep *rep)
+{
+       struct rpcrdma_mw *mw;
+
+       if (!(rep->rr_wc_flags & IB_WC_WITH_INVALIDATE))
+               return;
+
+       list_for_each_entry(mw, mws, mw_list)
+               if (mw->mw_handle == rep->rr_inv_rkey) {
+                       mw->mw_flags = RPCRDMA_MW_F_RI;
+                       break; /* only one invalidated MR per RPC */
+               }
+}
+
 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
 /* By convention, backchannel calls arrive via rdma_msg type
  * messages, and never populate the chunk lists. This makes
@@ -1006,13 +1024,13 @@ rpcrdma_reply_handler(struct work_struct *work)
        /* Sanity checking has passed. We are now committed
         * to complete this transaction.
         */
+       rpcrdma_mark_remote_invalidation(&req->rl_registered, rep);
        list_del_init(&rqst->rq_list);
+       req->rl_reply = rep;
        spin_unlock_bh(&xprt->transport_lock);
        dprintk("RPC:       %s: reply %p completes request %p (xid 0x%08x)\n",
                __func__, rep, req, be32_to_cpu(headerp->rm_xid));
 
-       /* from here on, the reply is no longer an orphan */
-       req->rl_reply = rep;
        xprt->reestablish_timeout = 0;
 
        if (headerp->rm_vers != rpcrdma_version)

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 3dbce9ac4327a89ea6d84f348041609c5d653b4a..a8be66d806dcc8ccc7171af8cfa0a74286c8f807 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -1187,6 +1187,7 @@ rpcrdma_get_mw(struct rpcrdma_xprt *r_xprt)
 
        if (!mw)
                goto out_nomws;
+       mw->mw_flags = 0;
        return mw;
 
 out_nomws:

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 1d66acf1a723e51efb0b2e91065122cd66856af7..2e027335fcbc7510a9342fbd091a8d31e27681cf 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -271,6 +271,7 @@ struct rpcrdma_mw {
        struct scatterlist      *mw_sg;
        int                     mw_nents;
        enum dma_data_direction mw_dir;
+       unsigned long           mw_flags;
        union {
                struct rpcrdma_fmr      fmr;
                struct rpcrdma_frmr     frmr;
@@ -282,6 +283,11 @@ struct rpcrdma_mw {
        struct list_head        mw_all;
 };
 
+/* mw_flags */
+enum {
+       RPCRDMA_MW_F_RI         = 1,
+};
+
 /*
  * struct rpcrdma_req -- structure central to the request/reply sequence.
  *