IB/mlx5: Expose correct max_sge_rd limit

mlx5 devices (Connect-IB, ConnectX-4, ConnectX-4-LX) has a limitation
where rdma read work queue entries cannot exceed 512 bytes.
A rdma_read wqe needs to fit in 512 bytes:
- wqe control segment (16 bytes)
- rdma segment (16 bytes)
- scatter elements (16 bytes each)

So max_sge_rd should be: (512 - 16 - 16) / 16 = 30.

Cc: linux-stable@vger.kernel.org
Reported-by: Christoph Hellwig <hch@lst.de>
Tested-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Sagi Grimberg <sagig@grimberg.me>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 5acf346e048e3bb45fc33d97eeecd1321bf2ef9d..049754f6e6382f9529fa13504224a4543778f24f 100644 (file)
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -530,7 +530,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
                     sizeof(struct mlx5_wqe_ctrl_seg)) /
                     sizeof(struct mlx5_wqe_data_seg);
        props->max_sge = min(max_rq_sg, max_sq_sg);
-       props->max_sge_rd = props->max_sge;
+       props->max_sge_rd          = MLX5_MAX_SGE_RD;
        props->max_cq              = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
        props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
        props->max_mr              = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);

diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 8156e3c9239ce6a5c6883040b1b6d1b619ad1169..b3575f392492de0a34ff9d3935c11ea6a8af8807 100644 (file)
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -392,6 +392,17 @@ enum {
        MLX5_CAP_OFF_CMDIF_CSUM         = 46,
 };
 
+enum {
+       /*
+        * Max wqe size for rdma read is 512 bytes, so this
+        * limits our max_sge_rd as the wqe needs to fit:
+        * - ctrl segment (16 bytes)
+        * - rdma segment (16 bytes)
+        * - scatter elements (16 bytes each)
+        */
+       MLX5_MAX_SGE_RD = (512 - 16 - 16) / 16
+};
+
 struct mlx5_inbox_hdr {
        __be16          opcode;
        u8              rsvd[4];