RDS: IB: split the mr registration and invalidation path
authorSantosh Shilimkar <santosh.shilimkar@oracle.com>
Tue, 8 Mar 2016 17:19:01 +0000 (09:19 -0800)
committerSantosh Shilimkar <santosh.shilimkar@oracle.com>
Mon, 2 Jan 2017 22:02:47 +0000 (14:02 -0800)
MR invalidation in RDS is done in background thread and not in
data path like registration. So break the dependency between them
which helps to remove the performance bottleneck.

Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>
net/rds/ib.h
net/rds/ib_cm.c
net/rds/ib_frmr.c

index f4e81214e70aeadc641a7f94ecb826f915fce5d6..f14c26d22b272b554be5cb375018bb26b2e0ffd5 100644 (file)
@@ -14,7 +14,8 @@
 
 #define RDS_IB_DEFAULT_RECV_WR         1024
 #define RDS_IB_DEFAULT_SEND_WR         256
-#define RDS_IB_DEFAULT_FR_WR           512
+#define RDS_IB_DEFAULT_FR_WR           256
+#define RDS_IB_DEFAULT_FR_INV_WR       256
 
 #define RDS_IB_DEFAULT_RETRY_COUNT     1
 
@@ -125,6 +126,7 @@ struct rds_ib_connection {
 
        /* To control the number of wrs from fastreg */
        atomic_t                i_fastreg_wrs;
+       atomic_t                i_fastunreg_wrs;
 
        /* interrupt handling */
        struct tasklet_struct   i_send_tasklet;
index b9da1e59ecc1f164ea3e95faf7789c6b30a78b1e..3002acf757666ab3bdc73e7862bc7024c8548c47 100644 (file)
@@ -382,7 +382,10 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
         * completion queue and send queue. This extra space is used for FRMR
         * registration and invalidation work requests
         */
-       fr_queue_space = (rds_ibdev->use_fastreg ? RDS_IB_DEFAULT_FR_WR : 0);
+       fr_queue_space = rds_ibdev->use_fastreg ?
+                        (RDS_IB_DEFAULT_FR_WR + 1) +
+                        (RDS_IB_DEFAULT_FR_INV_WR + 1)
+                        : 0;
 
        /* add the conn now so that connection establishment has the dev */
        rds_ib_add_conn(rds_ibdev, conn);
@@ -444,6 +447,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
        attr.send_cq = ic->i_send_cq;
        attr.recv_cq = ic->i_recv_cq;
        atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
+       atomic_set(&ic->i_fastunreg_wrs, RDS_IB_DEFAULT_FR_INV_WR);
 
        /*
         * XXX this can fail if max_*_wr is too large?  Are we supposed
@@ -766,7 +770,8 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
                wait_event(rds_ib_ring_empty_wait,
                           rds_ib_ring_empty(&ic->i_recv_ring) &&
                           (atomic_read(&ic->i_signaled_sends) == 0) &&
-                          (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));
+                          (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR) &&
+                          (atomic_read(&ic->i_fastunreg_wrs) == RDS_IB_DEFAULT_FR_INV_WR));
                tasklet_kill(&ic->i_send_tasklet);
                tasklet_kill(&ic->i_recv_tasklet);
 
index 66b3d6228a15f9431e6c0014e0407139eb3f2936..48332a6ed7383c51def7402dcfef1e581fa677f7 100644 (file)
@@ -241,8 +241,8 @@ static int rds_ib_post_inv(struct rds_ib_mr *ibmr)
        if (frmr->fr_state != FRMR_IS_INUSE)
                goto out;
 
-       while (atomic_dec_return(&ibmr->ic->i_fastreg_wrs) <= 0) {
-               atomic_inc(&ibmr->ic->i_fastreg_wrs);
+       while (atomic_dec_return(&ibmr->ic->i_fastunreg_wrs) <= 0) {
+               atomic_inc(&ibmr->ic->i_fastunreg_wrs);
                cpu_relax();
        }
 
@@ -261,7 +261,7 @@ static int rds_ib_post_inv(struct rds_ib_mr *ibmr)
        if (unlikely(ret)) {
                frmr->fr_state = FRMR_IS_STALE;
                frmr->fr_inv = false;
-               atomic_inc(&ibmr->ic->i_fastreg_wrs);
+               atomic_inc(&ibmr->ic->i_fastunreg_wrs);
                pr_err("RDS/IB: %s returned error(%d)\n", __func__, ret);
                goto out;
        }
@@ -289,9 +289,10 @@ void rds_ib_mr_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc)
        if (frmr->fr_inv) {
                frmr->fr_state = FRMR_IS_FREE;
                frmr->fr_inv = false;
+               atomic_inc(&ic->i_fastreg_wrs);
+       } else {
+               atomic_inc(&ic->i_fastunreg_wrs);
        }
-
-       atomic_inc(&ic->i_fastreg_wrs);
 }
 
 void rds_ib_unreg_frmr(struct list_head *list, unsigned int *nfreed,