IB/iser: Re-adjust CQ and QP send ring sizes to HW limits

Re-adjust max CQEs per CQ and max send_wr per QP according
to the resource limits supported by underlying hardware.

Signed-off-by: Minh Tran <minhduc.tran@emulex.com>
Signed-off-by: Jayamohan Kallickal <jayamohan.kallickal@emulex.com>
Acked-by: Sagi Grimberg <sagig@mellanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>

diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 20ca6a619476f065af6dbef57905897f717faf54..46028151a904f2aa07e21ceeec38064c70bde077 100644 (file)
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -569,6 +569,7 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
        struct Scsi_Host *shost;
        struct iser_conn *iser_conn = NULL;
        struct ib_conn *ib_conn;
+       u16 max_cmds;
 
        shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0);
        if (!shost)
@@ -586,6 +587,7 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
         */
        if (ep) {
                iser_conn = ep->dd_data;
+               max_cmds = iser_conn->max_cmds;
                ib_conn = &iser_conn->ib_conn;
                if (ib_conn->pi_support) {
                        u32 sig_caps = ib_conn->device->dev_attr.sig_prot_cap;
@@ -596,16 +598,18 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
                        else
                                scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
                }
+       } else {
+               max_cmds = ISER_DEF_XMIT_CMDS_MAX;
        }
 
        if (iscsi_host_add(shost, ep ?
                           ib_conn->device->ib_device->dma_device : NULL))
                goto free_host;
 
-       if (cmds_max > ISER_DEF_XMIT_CMDS_MAX) {
+       if (cmds_max > max_cmds) {
                iser_info("cmds_max changed from %u to %u\n",
-                         cmds_max, ISER_DEF_XMIT_CMDS_MAX);
-               cmds_max = ISER_DEF_XMIT_CMDS_MAX;
+                         cmds_max, max_cmds);
+               cmds_max = max_cmds;
        }
 
        cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,

diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index cd4174ca9a760dc70c0c369721407e1c2919a4c4..44d64b74c365ce9d5952269a7e00adaf714f61b5 100644 (file)
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -144,6 +144,11 @@
                                        ISER_MAX_TX_MISC_PDUS         + \
                                        ISER_MAX_RX_MISC_PDUS)
 
+#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr                      \
+                                        - ISER_MAX_TX_MISC_PDUS        \
+                                        - ISER_MAX_RX_MISC_PDUS) /     \
+                                        (1 + ISER_INFLIGHT_DATAOUTS))
+
 #define ISER_WC_BATCH_COUNT   16
 #define ISER_SIGNAL_CMD_COUNT 32
 
@@ -482,6 +487,7 @@ struct ib_conn {
  *                    to max number of post recvs
  * @qp_max_recv_dtos_mask: (qp_max_recv_dtos - 1)
  * @min_posted_rx:    (qp_max_recv_dtos >> 2)
+ * @max_cmds:         maximum cmds allowed for this connection
  * @name:             connection peer portal
  * @release_work:     deffered work for release job
  * @state_mutex:      protects iser onnection state
@@ -507,6 +513,7 @@ struct iser_conn {
        unsigned                     qp_max_recv_dtos;
        unsigned                     qp_max_recv_dtos_mask;
        unsigned                     min_posted_rx;
+       u16                          max_cmds;
        char                         name[ISER_OBJECT_NAME_SIZE];
        struct work_struct           release_work;
        struct mutex                 state_mutex;

diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 67225bb82bb50bc4b35ac8f350a750e5e3d96797..08e51e79a054b6f6e7cbe7b3e3e422c9343a4646 100644 (file)
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -76,7 +76,7 @@ static void iser_event_handler(struct ib_event_handler *handler,
 static int iser_create_device_ib_res(struct iser_device *device)
 {
        struct ib_device_attr *dev_attr = &device->dev_attr;
-       int ret, i;
+       int ret, i, max_cqe;
 
        ret = ib_query_device(device->ib_device, dev_attr);
        if (ret) {
@@ -106,9 +106,12 @@ static int iser_create_device_ib_res(struct iser_device *device)
 
        device->comps_used = min(ISER_MAX_CQ,
                                 device->ib_device->num_comp_vectors);
-       iser_info("using %d CQs, device %s supports %d vectors\n",
+
+       max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe);
+
+       iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n",
                  device->comps_used, device->ib_device->name,
-                 device->ib_device->num_comp_vectors);
+                 device->ib_device->num_comp_vectors, max_cqe);
 
        device->pd = ib_alloc_pd(device->ib_device);
        if (IS_ERR(device->pd))
@@ -122,7 +125,7 @@ static int iser_create_device_ib_res(struct iser_device *device)
                                        iser_cq_callback,
                                        iser_cq_event_callback,
                                        (void *)comp,
-                                       ISER_MAX_CQ_LEN, i);
+                                       max_cqe, i);
                if (IS_ERR(comp->cq)) {
                        comp->cq = NULL;
                        goto cq_err;
@@ -425,7 +428,10 @@ void iser_free_fastreg_pool(struct ib_conn *ib_conn)
  */
 static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
 {
+       struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
+                                                  ib_conn);
        struct iser_device      *device;
+       struct ib_device_attr *dev_attr;
        struct ib_qp_init_attr  init_attr;
        int                     ret = -ENOMEM;
        int index, min_index = 0;
@@ -433,6 +439,7 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
        BUG_ON(ib_conn->device == NULL);
 
        device = ib_conn->device;
+       dev_attr = &device->dev_attr;
 
        memset(&init_attr, 0, sizeof init_attr);
 
@@ -460,8 +467,20 @@ static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
        if (ib_conn->pi_support) {
                init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
                init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
+               iser_conn->max_cmds =
+                       ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS);
        } else {
-               init_attr.cap.max_send_wr  = ISER_QP_MAX_REQ_DTOS + 1;
+               if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
+                       init_attr.cap.max_send_wr  = ISER_QP_MAX_REQ_DTOS + 1;
+                       iser_conn->max_cmds =
+                               ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
+               } else {
+                       init_attr.cap.max_send_wr = dev_attr->max_qp_wr;
+                       iser_conn->max_cmds =
+                               ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr);
+                       iser_dbg("device %s supports max_send_wr %d\n",
+                                device->ib_device->name, dev_attr->max_qp_wr);
+               }
        }
 
        ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);