RDMA/mlx5: set UMR wqe fence according to HCA cap

Cache the needed umr_fence and set the wqe ctrl segmennt
accordingly.

Signed-off-by: Max Gurtovoy <maxg@mellanox.com>
Acked-by: Leon Romanovsky <leon@kernel.org>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index d45772da09635c2164f4cef8bcf5255c17fe8cff..0c79983c8b1a0a4e6189fb5e02439ff3522a9098 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -2979,6 +2979,18 @@ error_0:
        return ret;
 }
 
+static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
+{
+       switch (umr_fence_cap) {
+       case MLX5_CAP_UMR_FENCE_NONE:
+               return MLX5_FENCE_MODE_NONE;
+       case MLX5_CAP_UMR_FENCE_SMALL:
+               return MLX5_FENCE_MODE_INITIATOR_SMALL;
+       default:
+               return MLX5_FENCE_MODE_STRONG_ORDERING;
+       }
+}
+
 static int create_dev_resources(struct mlx5_ib_resources *devr)
 {
        struct ib_srq_init_attr attr;
@@ -3693,6 +3705,8 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
        mlx5_ib_internal_fill_odp_caps(dev);
 
+       dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
+
        if (MLX5_CAP_GEN(mdev, imaicl)) {
                dev->ib_dev.alloc_mw            = mlx5_ib_alloc_mw;
                dev->ib_dev.dealloc_mw          = mlx5_ib_dealloc_mw;

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 38c877bc45e592dbbe9670b3c9b624b90b6df30d..bdcf25410c99df7f57e280f2672eb86b8e3205fd 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -349,7 +349,7 @@ struct mlx5_ib_qp {
        struct mlx5_ib_wq       rq;
 
        u8                      sq_signal_bits;
-       u8                      fm_cache;
+       u8                      next_fence;
        struct mlx5_ib_wq       sq;
 
        /* serialize qp state modifications
@@ -654,6 +654,7 @@ struct mlx5_ib_dev {
        struct mlx5_ib_port     *port;
        struct mlx5_sq_bfreg     bfreg;
        struct mlx5_sq_bfreg     fp_bfreg;
+       u8                              umr_fence;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)

diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 93959e1e43a3da5e2f71a89c8c12bdc61368e3e0..ebb6768684de372d755cff4c6f92c768edb9045b 100644 (file)
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -3738,24 +3738,6 @@ static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
        }
 }
 
-static u8 get_fence(u8 fence, struct ib_send_wr *wr)
-{
-       if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&
-                    wr->send_flags & IB_SEND_FENCE))
-               return MLX5_FENCE_MODE_STRONG_ORDERING;
-
-       if (unlikely(fence)) {
-               if (wr->send_flags & IB_SEND_FENCE)
-                       return MLX5_FENCE_MODE_SMALL_AND_FENCE;
-               else
-                       return fence;
-       } else if (unlikely(wr->send_flags & IB_SEND_FENCE)) {
-               return MLX5_FENCE_MODE_FENCE;
-       }
-
-       return 0;
-}
-
 static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
                     struct mlx5_wqe_ctrl_seg **ctrl,
                     struct ib_send_wr *wr, unsigned *idx,
@@ -3784,8 +3766,7 @@ static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
 static void finish_wqe(struct mlx5_ib_qp *qp,
                       struct mlx5_wqe_ctrl_seg *ctrl,
                       u8 size, unsigned idx, u64 wr_id,
-                      int nreq, u8 fence, u8 next_fence,
-                      u32 mlx5_opcode)
+                      int nreq, u8 fence, u32 mlx5_opcode)
 {
        u8 opmod = 0;
 
@@ -3793,7 +3774,6 @@ static void finish_wqe(struct mlx5_ib_qp *qp,
                                             mlx5_opcode | ((u32)opmod << 24));
        ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));
        ctrl->fm_ce_se |= fence;
-       qp->fm_cache = next_fence;
        if (unlikely(qp->wq_sig))
                ctrl->signature = wq_sig(ctrl);
 
@@ -3853,7 +3833,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                        goto out;
                }
 
-               fence = qp->fm_cache;
                num_sge = wr->num_sge;
                if (unlikely(num_sge > qp->sq.max_gs)) {
                        mlx5_ib_warn(dev, "\n");
@@ -3870,6 +3849,19 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                        goto out;
                }
 
+               if (wr->opcode == IB_WR_LOCAL_INV ||
+                   wr->opcode == IB_WR_REG_MR) {
+                       fence = dev->umr_fence;
+                       next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
+               } else if (wr->send_flags & IB_SEND_FENCE) {
+                       if (qp->next_fence)
+                               fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;
+                       else
+                               fence = MLX5_FENCE_MODE_FENCE;
+               } else {
+                       fence = qp->next_fence;
+               }
+
                switch (ibqp->qp_type) {
                case IB_QPT_XRC_INI:
                        xrc = seg;
@@ -3896,7 +3888,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                                goto out;
 
                        case IB_WR_LOCAL_INV:
-                               next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
                                qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;
                                ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);
                                set_linv_wr(qp, &seg, &size);
@@ -3904,7 +3895,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                                break;
 
                        case IB_WR_REG_MR:
-                               next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
                                qp->sq.wr_data[idx] = IB_WR_REG_MR;
                                ctrl->imm = cpu_to_be32(reg_wr(wr)->key);
                                err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
@@ -3927,9 +3917,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                                        goto out;
                                }
 
-                               finish_wqe(qp, ctrl, size, idx, wr->wr_id,
-                                          nreq, get_fence(fence, wr),
-                                          next_fence, MLX5_OPCODE_UMR);
+                               finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
+                                          fence, MLX5_OPCODE_UMR);
                                /*
                                 * SET_PSV WQEs are not signaled and solicited
                                 * on error
@@ -3954,9 +3943,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                                        goto out;
                                }
 
-                               finish_wqe(qp, ctrl, size, idx, wr->wr_id,
-                                          nreq, get_fence(fence, wr),
-                                          next_fence, MLX5_OPCODE_SET_PSV);
+                               finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
+                                          fence, MLX5_OPCODE_SET_PSV);
                                err = begin_wqe(qp, &seg, &ctrl, wr,
                                                &idx, &size, nreq);
                                if (err) {
@@ -3966,7 +3954,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                                        goto out;
                                }
 
-                               next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
                                err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,
                                                 mr->sig->psv_wire.psv_idx, &seg,
                                                 &size);
@@ -3976,9 +3963,9 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                                        goto out;
                                }
 
-                               finish_wqe(qp, ctrl, size, idx, wr->wr_id,
-                                          nreq, get_fence(fence, wr),
-                                          next_fence, MLX5_OPCODE_SET_PSV);
+                               finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
+                                          fence, MLX5_OPCODE_SET_PSV);
+                               qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;
                                num_sge = 0;
                                goto skip_psv;
 
@@ -4089,8 +4076,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
                        }
                }
 
-               finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
-                          get_fence(fence, wr), next_fence,
+               qp->next_fence = next_fence;
+               finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, fence,
                           mlx5_ib_opcode[wr->opcode]);
 skip_psv:
                if (0)