IB/hns: Fix the bug of polling cq failed for loopback Qps

In hip06 SoC, RoCE driver creates 8 reserved loopback QPs to
ensure zero wqe when free mr. However, if the enabled phy
port number is less than 6, it will fail in polling cqe with
8 reserved loopback QPs.

In order to solve this problem, the number of loopback Qps
will be adjusted based on the number of enabled phy port.

Signed-off-by: Shaobo Xu <xushaobo2@huawei.com>
Signed-off-by: Lijun Ou <oulijun@huawei.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index 729f8cc8738be2a23b2166f5011ae36e7ac844ed..c291b2a4c7cf9ff8f24f6336cefdfe6df5a330f0 100644 (file)
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -661,9 +661,11 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
        union ib_gid dgid;
        u64 subnet_prefix;
        int attr_mask = 0;
-       int i;
+       int i, j;
        int ret;
+       u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 };
        u8 phy_port;
+       u8 port = 0;
        u8 sl;
 
        priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
@@ -709,11 +711,27 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
        attr.rnr_retry          = 7;
        attr.timeout            = 0x12;
        attr.path_mtu           = IB_MTU_256;
+       attr.ah_attr.type       = RDMA_AH_ATTR_TYPE_ROCE;
        rdma_ah_set_grh(&attr.ah_attr, NULL, 0, 0, 1, 0);
        rdma_ah_set_static_rate(&attr.ah_attr, 3);
 
        subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
        for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
+               phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
+                               (i % HNS_ROCE_MAX_PORTS);
+               sl = i / HNS_ROCE_MAX_PORTS;
+
+               for (j = 0; j < caps->num_ports; j++) {
+                       if (hr_dev->iboe.phy_port[j] == phy_port) {
+                               queue_en[i] = 1;
+                               port = j;
+                               break;
+                       }
+               }
+
+               if (!queue_en[i])
+                       continue;
+
                free_mr->mr_free_qp[i] = hns_roce_v1_create_lp_qp(hr_dev, pd);
                if (IS_ERR(free_mr->mr_free_qp[i])) {
                        dev_err(dev, "Create loop qp failed!\n");
@@ -721,15 +739,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
                }
                hr_qp = free_mr->mr_free_qp[i];
 
-               sl = i / caps->num_ports;
-
-               if (caps->num_ports == HNS_ROCE_MAX_PORTS)
-                       phy_port = (i >= HNS_ROCE_MAX_PORTS) ? (i - 2) :
-                               (i % caps->num_ports);
-               else
-                       phy_port = i % caps->num_ports;
-
-               hr_qp->port             = phy_port + 1;
+               hr_qp->port             = port;
                hr_qp->phy_port         = phy_port;
                hr_qp->ibqp.qp_type     = IB_QPT_RC;
                hr_qp->ibqp.device      = &hr_dev->ib_dev;
@@ -739,23 +749,22 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
                hr_qp->ibqp.recv_cq     = cq;
                hr_qp->ibqp.send_cq     = cq;
 
-               rdma_ah_set_port_num(&attr.ah_attr, phy_port + 1);
-               rdma_ah_set_sl(&attr.ah_attr, phy_port + 1);
-               attr.port_num           = phy_port + 1;
+               rdma_ah_set_port_num(&attr.ah_attr, port + 1);
+               rdma_ah_set_sl(&attr.ah_attr, sl);
+               attr.port_num           = port + 1;
 
                attr.dest_qp_num        = hr_qp->qpn;
                memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr),
-                      hr_dev->dev_addr[phy_port],
+                      hr_dev->dev_addr[port],
                       MAC_ADDR_OCTET_NUM);
 
                memcpy(&dgid.raw, &subnet_prefix, sizeof(u64));
-               memcpy(&dgid.raw[8], hr_dev->dev_addr[phy_port], 3);
-               memcpy(&dgid.raw[13], hr_dev->dev_addr[phy_port] + 3, 3);
+               memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3);
+               memcpy(&dgid.raw[13], hr_dev->dev_addr[port] + 3, 3);
                dgid.raw[11] = 0xff;
                dgid.raw[12] = 0xfe;
                dgid.raw[8] ^= 2;
                rdma_ah_set_dgid_raw(&attr.ah_attr, dgid.raw);
-               attr_mask |= IB_QP_PORT;
 
                ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
                                            IB_QPS_RESET, IB_QPS_INIT);
@@ -812,6 +821,9 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
 
        for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
                hr_qp = free_mr->mr_free_qp[i];
+               if (!hr_qp)
+                       continue;
+
                ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp);
                if (ret)
                        dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
@@ -963,7 +975,7 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
                msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies;
        int i;
        int ret;
-       int ne;
+       int ne = 0;
 
        mr_work = container_of(work, struct hns_roce_mr_free_work, work);
        hr_mr = (struct hns_roce_mr *)mr_work->mr;
@@ -976,6 +988,10 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
 
        for (i = 0; i < HNS_ROCE_V1_RESV_QP; i++) {
                hr_qp = free_mr->mr_free_qp[i];
+               if (!hr_qp)
+                       continue;
+               ne++;
+
                ret = hns_roce_v1_send_lp_wqe(hr_qp);
                if (ret) {
                        dev_err(dev,
@@ -985,7 +1001,6 @@ static void hns_roce_v1_mr_free_work_fn(struct work_struct *work)
                }
        }
 
-       ne = HNS_ROCE_V1_RESV_QP;
        do {
                ret = hns_roce_v1_poll_cq(&mr_free_cq->ib_cq, ne, wc);
                if (ret < 0) {