IB/core: Enhance ib_map_mr_sg()

The SRP initiator allows to set max_sectors to a value that exceeds
the largest amount of data that can be mapped at once with an mlx4
HCA using fast registration and a page size of 4 KB. Hence modify
ib_map_mr_sg() such that it can map partial sg-elements. If an
sg-element has been mapped partially, let the caller know
which fraction has been mapped by adjusting *sg_offset.

Signed-off-by: Bart Van Assche <bart.vanassche@sandisk.com>
Tested-by: Laurence Oberman <loberman@redhat.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index 6fc50bf79afe2d16bbff10085d02a29874f81fa0..1eb9b1294a6383c689363bb02a713ee1cd660037 100644 (file)
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -92,7 +92,7 @@ static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,
                reg->inv_wr.next = NULL;
        }
 
-       ret = ib_map_mr_sg(reg->mr, sg, nents, offset, PAGE_SIZE);
+       ret = ib_map_mr_sg(reg->mr, sg, nents, &offset, PAGE_SIZE);
        if (ret < nents) {
                ib_mr_pool_put(qp, &qp->rdma_mrs, reg->mr);
                return -EINVAL;

diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 3d7b266a2dcb5c2afc9c86a564f66d0d4ade1ec9..1d7d4cf442e3c9646b6824e943e7e649c873d104 100644 (file)
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -1655,7 +1655,7 @@ EXPORT_SYMBOL(ib_set_vf_guid);
  * is ready for registration.
  */
 int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset, unsigned int page_size)
+                unsigned int *sg_offset, unsigned int page_size)
 {
        if (unlikely(!mr->device->map_mr_sg))
                return -ENOSYS;
@@ -1672,7 +1672,10 @@ EXPORT_SYMBOL(ib_map_mr_sg);
  * @mr:            memory region
  * @sgl:           dma mapped scatterlist
  * @sg_nents:      number of entries in sg
- * @sg_offset:     offset in bytes into sg
+ * @sg_offset_p:   IN:  start offset in bytes into sg
+ *                 OUT: offset in bytes for element n of the sg of the first
+ *                      byte that has not been processed where n is the return
+ *                      value of this function.
  * @set_page:      driver page assignment function pointer
  *
  * Core service helper for drivers to convert the largest
@@ -1684,19 +1687,24 @@ EXPORT_SYMBOL(ib_map_mr_sg);
  * a page vector.
  */
 int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
-               unsigned int sg_offset, int (*set_page)(struct ib_mr *, u64))
+               unsigned int *sg_offset_p, int (*set_page)(struct ib_mr *, u64))
 {
        struct scatterlist *sg;
        u64 last_end_dma_addr = 0;
+       unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
        unsigned int last_page_off = 0;
        u64 page_mask = ~((u64)mr->page_size - 1);
        int i, ret;
 
+       if (unlikely(sg_nents <= 0 || sg_offset > sg_dma_len(&sgl[0])))
+               return -EINVAL;
+
        mr->iova = sg_dma_address(&sgl[0]) + sg_offset;
        mr->length = 0;
 
        for_each_sg(sgl, sg, sg_nents, i) {
                u64 dma_addr = sg_dma_address(sg) + sg_offset;
+               u64 prev_addr = dma_addr;
                unsigned int dma_len = sg_dma_len(sg) - sg_offset;
                u64 end_dma_addr = dma_addr + dma_len;
                u64 page_addr = dma_addr & page_mask;
@@ -1721,8 +1729,14 @@ int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
 
                do {
                        ret = set_page(mr, page_addr);
-                       if (unlikely(ret < 0))
-                               return i ? : ret;
+                       if (unlikely(ret < 0)) {
+                               sg_offset = prev_addr - sg_dma_address(sg);
+                               mr->length += prev_addr - dma_addr;
+                               if (sg_offset_p)
+                                       *sg_offset_p = sg_offset;
+                               return i || sg_offset ? i : ret;
+                       }
+                       prev_addr = page_addr;
 next_page:
                        page_addr += mr->page_size;
                } while (page_addr < end_dma_addr);
@@ -1734,6 +1748,8 @@ next_page:
                sg_offset = 0;
        }
 
+       if (sg_offset_p)
+               *sg_offset_p = 0;
        return i;
 }
 EXPORT_SYMBOL(ib_sg_to_pages);

diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 608aa0c16dc3e9631f3d8ffc31f3ed0e618687a8..47cb927a0dd665bd0dfb0f85508650aa97aaa7b7 100644 (file)
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -784,7 +784,7 @@ static int iwch_set_page(struct ib_mr *ibmr, u64 addr)
 }
 
 static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-               int sg_nents, unsigned sg_offset)
+                         int sg_nents, unsigned int *sg_offset)
 {
        struct iwch_mr *mhp = to_iwch_mr(ibmr);
 

diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 067cb3f909c16b323f5b4fa48be376d675610546..1ff3ba8ab67b58f135607009c65e304328de89fa 100644 (file)
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -918,7 +918,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
                            enum ib_mr_type mr_type,
                            u32 max_num_sg);
 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset);
+                  unsigned int *sg_offset);
 int c4iw_dealloc_mw(struct ib_mw *mw);
 struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
                            struct ib_udata *udata);

diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 38afb3d2dd9284475dd681ae28a4f2e08dcbbb45..83960df6fe602595c9b0910094aae76e635f6be6 100644 (file)
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -691,7 +691,7 @@ static int c4iw_set_page(struct ib_mr *ibmr, u64 addr)
 }
 
 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset)
+                  unsigned int *sg_offset)
 {
        struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
 

diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 825430e376fc5ea861000341bf11201ca53fa93d..4a740f7a0519bcac97d35b86844ae844727daeb4 100644 (file)
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -1574,7 +1574,7 @@ static int i40iw_set_page(struct ib_mr *ibmr, u64 addr)
  * @sg_nents: number of sg pages
  */
 static int i40iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-                          int sg_nents, unsigned int sg_offset)
+                          int sg_nents, unsigned int *sg_offset)
 {
        struct i40iw_mr *iwmr = to_iwmr(ibmr);
 

diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index ba328177eae9930ec197e66dc86cbb5533eaf838..6c5ac5d8f32ffcfafbb4dd7a4b9202b4049b95b5 100644 (file)
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -718,7 +718,7 @@ struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
                               enum ib_mr_type mr_type,
                               u32 max_num_sg);
 int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset);
+                     unsigned int *sg_offset);
 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
 int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
 struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,

diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index b04f6238e7e2fb4277cd7591033c68da83e62d24..631272172a0b9793c0074e6ab9c00a4b9f2e4665 100644 (file)
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -529,7 +529,7 @@ static int mlx4_set_page(struct ib_mr *ibmr, u64 addr)
 }
 
 int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset)
+                     unsigned int *sg_offset)
 {
        struct mlx4_ib_mr *mr = to_mmr(ibmr);
        int rc;

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 8c835b2be39e549404467d86f90eaac8ed314611..f05cf57f874cfee8189461922756728643173da8 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -713,7 +713,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
                               enum ib_mr_type mr_type,
                               u32 max_num_sg);
 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset);
+                     unsigned int *sg_offset);
 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
                        const struct ib_wc *in_wc, const struct ib_grh *in_grh,
                        const struct ib_mad_hdr *in, size_t in_mad_size,

diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index b678eac0f8b3f400280216b9f2e81d008bc2b2a1..8cf2ce50511f935e6043bea4ebaa4af3a980661c 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1752,10 +1752,11 @@ static int
 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
                   struct scatterlist *sgl,
                   unsigned short sg_nents,
-                  unsigned int sg_offset)
+                  unsigned int *sg_offset_p)
 {
        struct scatterlist *sg = sgl;
        struct mlx5_klm *klms = mr->descs;
+       unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
        u32 lkey = mr->ibmr.pd->local_dma_lkey;
        int i;
 
@@ -1774,6 +1775,9 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
                sg_offset = 0;
        }
 
+       if (sg_offset_p)
+               *sg_offset_p = sg_offset;
+
        return i;
 }
 
@@ -1792,7 +1796,7 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
 }
 
 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset)
+                     unsigned int *sg_offset)
 {
        struct mlx5_ib_mr *mr = to_mmr(ibmr);
        int n;

diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 698aab65a28635a5aa76ab2ae61eedfe23d99ab4..4ebea4c8c9b5164ef1cf31ad6f4f8f5fa2c5e95d 100644 (file)
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -403,7 +403,7 @@ static int nes_set_page(struct ib_mr *ibmr, u64 addr)
 }
 
 static int nes_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
-               int sg_nents, unsigned int sg_offset)
+                        int sg_nents, unsigned int *sg_offset)
 {
        struct nes_mr *nesmr = to_nesmr(ibmr);
 

diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 9ddd55022bafc0bbd6a262ca1e64fd264413f374..b1a3d91fe8b94fe292fbe09a4c3820a8592e9f6e 100644 (file)
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -3082,7 +3082,7 @@ static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)
 }
 
 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset)
+                    unsigned int *sg_offset)
 {
        struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
 

diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
index b290e5dfc5f1f87f82f5e50cdb4343f63fa4259e..704ef1e9271b74b62ec14e5cf7178014fed8df72 100644 (file)
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
@@ -123,6 +123,6 @@ struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd,
                              enum ib_mr_type mr_type,
                              u32 max_num_sg);
 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
-               unsigned sg_offset);
+                    unsigned int *sg_offset);
 
 #endif                         /* __OCRDMA_VERBS_H__ */

diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 44cc85f206f3410fc25d01c5c3b94f5ab00caac6..90be568934146aaf55bd01bd801f177c12bcfd2a 100644 (file)
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -236,7 +236,7 @@ int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task,
        page_vec->npages = 0;
        page_vec->fake_mr.page_size = SIZE_4K;
        plen = ib_sg_to_pages(&page_vec->fake_mr, mem->sg,
-                             mem->size, 0, iser_set_page);
+                             mem->size, NULL, iser_set_page);
        if (unlikely(plen < mem->size)) {
                iser_err("page vec too short to hold this SG\n");
                iser_data_buf_dump(mem, device->ib_device);
@@ -446,7 +446,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
 
        ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
 
-       n = ib_map_mr_sg(mr, mem->sg, mem->size, 0, SIZE_4K);
+       n = ib_map_mr_sg(mr, mem->sg, mem->size, NULL, SIZE_4K);
        if (unlikely(n != mem->size)) {
                iser_err("failed to map sg (%d/%d)\n",
                         n, mem->size);

diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 6b9c5688e26a1f130c7ea5e590c6bdd0164f8d43..54f4c131089761a0e7d60746652c8acb022998e5 100644 (file)
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -1329,7 +1329,7 @@ static int srp_map_finish_fr(struct srp_map_state *state,
        rkey = ib_inc_rkey(desc->mr->rkey);
        ib_update_fast_reg_key(desc->mr, rkey);
 
-       n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, 0, dev->mr_page_size);
+       n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, NULL, dev->mr_page_size);
        if (unlikely(n < 0)) {
                srp_fr_pool_put(ch->fr_pool, &desc, 1);
                pr_debug("%s: ib_map_mr_sg(%d) returned %d.\n",

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 544c55b4c84a93c05f47c7638c6e67da7102bd40..56bb0f39ce794f9c19b8df3529140178beb2d430 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -1848,7 +1848,7 @@ struct ib_device {
        int                        (*map_mr_sg)(struct ib_mr *mr,
                                                struct scatterlist *sg,
                                                int sg_nents,
-                                               unsigned sg_offset);
+                                               unsigned int *sg_offset);
        struct ib_mw *             (*alloc_mw)(struct ib_pd *pd,
                                               enum ib_mw_type type,
                                               struct ib_udata *udata);
@@ -3145,11 +3145,11 @@ struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u8 port,
                                            const struct sockaddr *addr);
 
 int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset, unsigned int page_size);
+                unsigned int *sg_offset, unsigned int page_size);
 
 static inline int
 ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
-               unsigned int sg_offset, unsigned int page_size)
+                 unsigned int *sg_offset, unsigned int page_size)
 {
        int n;
 
@@ -3160,7 +3160,7 @@ ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
 }
 
 int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
-               unsigned int sg_offset, int (*set_page)(struct ib_mr *, u64));
+               unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));
 
 void ib_drain_rq(struct ib_qp *qp);
 void ib_drain_sq(struct ib_qp *qp);

diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index 3274a4a33231713d290c22c542ac8ea30e1889b5..94c3fa910b85e5e4a53109a2121dd9be51067c49 100644 (file)
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -421,7 +421,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
                return -ENOMEM;
        }
 
-       n = ib_map_mr_sg(mr, frmr->sg, frmr->sg_nents, 0, PAGE_SIZE);
+       n = ib_map_mr_sg(mr, frmr->sg, frmr->sg_nents, NULL, PAGE_SIZE);
        if (unlikely(n != frmr->sg_nents)) {
                pr_err("RPC:       %s: failed to map mr %p (%u/%u)\n",
                       __func__, frmr->fr_mr, n, frmr->sg_nents);

diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 19a74e95cd381c40469cfc773e76358c8a59e2a1..fbe7444e7de6ab05ee1c6e91d09b03eca7ba1f3f 100644 (file)
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -281,7 +281,7 @@ int rdma_read_chunk_frmr(struct svcxprt_rdma *xprt,
        }
        atomic_inc(&xprt->sc_dma_used);
 
-       n = ib_map_mr_sg(frmr->mr, frmr->sg, frmr->sg_nents, 0, PAGE_SIZE);
+       n = ib_map_mr_sg(frmr->mr, frmr->sg, frmr->sg_nents, NULL, PAGE_SIZE);
        if (unlikely(n != frmr->sg_nents)) {
                pr_err("svcrdma: failed to map mr %p (%d/%d elements)\n",
                       frmr->mr, n, frmr->sg_nents);