2 * Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of EITHER the GNU General Public License
6 * version 2 as published by the Free Software Foundation or the BSD
7 * 2-Clause License. This program is distributed in the hope that it
8 * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
9 * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
10 * See the GNU General Public License version 2 for more details at
11 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
13 * You should have received a copy of the GNU General Public License
14 * along with this program available in the file COPYING in the main
15 * directory of this source tree.
17 * The BSD 2-Clause License
19 * Redistribution and use in source and binary forms, with or
20 * without modification, are permitted provided that the following
23 * - Redistributions of source code must retain the above
24 * copyright notice, this list of conditions and the following
27 * - Redistributions in binary form must reproduce the above
28 * copyright notice, this list of conditions and the following
29 * disclaimer in the documentation and/or other materials
30 * provided with the distribution.
32 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
33 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
34 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
37 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
38 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
39 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
41 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
43 * OF THE POSSIBILITY OF SUCH DAMAGE.
48 #include <linux/wait.h>
49 #include <rdma/ib_addr.h>
50 #include <rdma/ib_smi.h>
51 #include <rdma/ib_user_verbs.h>
55 static inline void get_cqs(struct pvrdma_qp
*qp
, struct pvrdma_cq
**send_cq
,
56 struct pvrdma_cq
**recv_cq
)
58 *send_cq
= to_vcq(qp
->ibqp
.send_cq
);
59 *recv_cq
= to_vcq(qp
->ibqp
.recv_cq
);
62 static void pvrdma_lock_cqs(struct pvrdma_cq
*scq
, struct pvrdma_cq
*rcq
,
63 unsigned long *scq_flags
,
64 unsigned long *rcq_flags
)
65 __acquires(scq
->cq_lock
) __acquires(rcq
->cq_lock
)
68 spin_lock_irqsave(&scq
->cq_lock
, *scq_flags
);
69 __acquire(rcq
->cq_lock
);
70 } else if (scq
->cq_handle
< rcq
->cq_handle
) {
71 spin_lock_irqsave(&scq
->cq_lock
, *scq_flags
);
72 spin_lock_irqsave_nested(&rcq
->cq_lock
, *rcq_flags
,
73 SINGLE_DEPTH_NESTING
);
75 spin_lock_irqsave(&rcq
->cq_lock
, *rcq_flags
);
76 spin_lock_irqsave_nested(&scq
->cq_lock
, *scq_flags
,
77 SINGLE_DEPTH_NESTING
);
81 static void pvrdma_unlock_cqs(struct pvrdma_cq
*scq
, struct pvrdma_cq
*rcq
,
82 unsigned long *scq_flags
,
83 unsigned long *rcq_flags
)
84 __releases(scq
->cq_lock
) __releases(rcq
->cq_lock
)
87 __release(rcq
->cq_lock
);
88 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
89 } else if (scq
->cq_handle
< rcq
->cq_handle
) {
90 spin_unlock_irqrestore(&rcq
->cq_lock
, *rcq_flags
);
91 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
93 spin_unlock_irqrestore(&scq
->cq_lock
, *scq_flags
);
94 spin_unlock_irqrestore(&rcq
->cq_lock
, *rcq_flags
);
98 static void pvrdma_reset_qp(struct pvrdma_qp
*qp
)
100 struct pvrdma_cq
*scq
, *rcq
;
101 unsigned long scq_flags
, rcq_flags
;
104 get_cqs(qp
, &scq
, &rcq
);
105 pvrdma_lock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
107 _pvrdma_flush_cqe(qp
, scq
);
109 _pvrdma_flush_cqe(qp
, rcq
);
111 pvrdma_unlock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
114 * Reset queuepair. The checks are because usermode queuepairs won't
115 * have kernel ringstates.
118 atomic_set(&qp
->rq
.ring
->cons_head
, 0);
119 atomic_set(&qp
->rq
.ring
->prod_tail
, 0);
122 atomic_set(&qp
->sq
.ring
->cons_head
, 0);
123 atomic_set(&qp
->sq
.ring
->prod_tail
, 0);
127 static int pvrdma_set_rq_size(struct pvrdma_dev
*dev
,
128 struct ib_qp_cap
*req_cap
,
129 struct pvrdma_qp
*qp
)
131 if (req_cap
->max_recv_wr
> dev
->dsr
->caps
.max_qp_wr
||
132 req_cap
->max_recv_sge
> dev
->dsr
->caps
.max_sge
) {
133 dev_warn(&dev
->pdev
->dev
, "recv queue size invalid\n");
137 qp
->rq
.wqe_cnt
= roundup_pow_of_two(max(1U, req_cap
->max_recv_wr
));
138 qp
->rq
.max_sg
= roundup_pow_of_two(max(1U, req_cap
->max_recv_sge
));
141 req_cap
->max_recv_wr
= qp
->rq
.wqe_cnt
;
142 req_cap
->max_recv_sge
= qp
->rq
.max_sg
;
144 qp
->rq
.wqe_size
= roundup_pow_of_two(sizeof(struct pvrdma_rq_wqe_hdr
) +
145 sizeof(struct pvrdma_sge
) *
147 qp
->npages_recv
= (qp
->rq
.wqe_cnt
* qp
->rq
.wqe_size
+ PAGE_SIZE
- 1) /
153 static int pvrdma_set_sq_size(struct pvrdma_dev
*dev
, struct ib_qp_cap
*req_cap
,
154 struct pvrdma_qp
*qp
)
156 if (req_cap
->max_send_wr
> dev
->dsr
->caps
.max_qp_wr
||
157 req_cap
->max_send_sge
> dev
->dsr
->caps
.max_sge
) {
158 dev_warn(&dev
->pdev
->dev
, "send queue size invalid\n");
162 qp
->sq
.wqe_cnt
= roundup_pow_of_two(max(1U, req_cap
->max_send_wr
));
163 qp
->sq
.max_sg
= roundup_pow_of_two(max(1U, req_cap
->max_send_sge
));
166 req_cap
->max_send_wr
= qp
->sq
.wqe_cnt
;
167 req_cap
->max_send_sge
= qp
->sq
.max_sg
;
169 qp
->sq
.wqe_size
= roundup_pow_of_two(sizeof(struct pvrdma_sq_wqe_hdr
) +
170 sizeof(struct pvrdma_sge
) *
172 /* Note: one extra page for the header. */
173 qp
->npages_send
= PVRDMA_QP_NUM_HEADER_PAGES
+
174 (qp
->sq
.wqe_cnt
* qp
->sq
.wqe_size
+ PAGE_SIZE
- 1) /
181 * pvrdma_create_qp - create queue pair
182 * @pd: protection domain
183 * @init_attr: queue pair attributes
186 * @return: the ib_qp pointer on success, otherwise returns an errno.
188 struct ib_qp
*pvrdma_create_qp(struct ib_pd
*pd
,
189 struct ib_qp_init_attr
*init_attr
,
190 struct ib_udata
*udata
)
192 struct pvrdma_qp
*qp
= NULL
;
193 struct pvrdma_dev
*dev
= to_vdev(pd
->device
);
194 union pvrdma_cmd_req req
;
195 union pvrdma_cmd_resp rsp
;
196 struct pvrdma_cmd_create_qp
*cmd
= &req
.create_qp
;
197 struct pvrdma_cmd_create_qp_resp
*resp
= &rsp
.create_qp_resp
;
198 struct pvrdma_create_qp ucmd
;
202 if (init_attr
->create_flags
) {
203 dev_warn(&dev
->pdev
->dev
,
204 "invalid create queuepair flags %#x\n",
205 init_attr
->create_flags
);
206 return ERR_PTR(-EINVAL
);
209 if (init_attr
->qp_type
!= IB_QPT_RC
&&
210 init_attr
->qp_type
!= IB_QPT_UD
&&
211 init_attr
->qp_type
!= IB_QPT_GSI
) {
212 dev_warn(&dev
->pdev
->dev
, "queuepair type %d not supported\n",
214 return ERR_PTR(-EINVAL
);
217 if (!atomic_add_unless(&dev
->num_qps
, 1, dev
->dsr
->caps
.max_qp
))
218 return ERR_PTR(-ENOMEM
);
220 switch (init_attr
->qp_type
) {
222 if (init_attr
->port_num
== 0 ||
223 init_attr
->port_num
> pd
->device
->phys_port_cnt
||
225 dev_warn(&dev
->pdev
->dev
, "invalid queuepair attrs\n");
232 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
238 spin_lock_init(&qp
->sq
.lock
);
239 spin_lock_init(&qp
->rq
.lock
);
240 mutex_init(&qp
->mutex
);
241 atomic_set(&qp
->refcnt
, 1);
242 init_waitqueue_head(&qp
->wait
);
244 qp
->state
= IB_QPS_RESET
;
246 if (pd
->uobject
&& udata
) {
247 dev_dbg(&dev
->pdev
->dev
,
248 "create queuepair from user space\n");
250 if (ib_copy_from_udata(&ucmd
, udata
, sizeof(ucmd
))) {
255 /* set qp->sq.wqe_cnt, shift, buf_size.. */
256 qp
->rumem
= ib_umem_get(pd
->uobject
->context
,
258 ucmd
.rbuf_size
, 0, 0);
259 if (IS_ERR(qp
->rumem
)) {
260 ret
= PTR_ERR(qp
->rumem
);
264 qp
->sumem
= ib_umem_get(pd
->uobject
->context
,
266 ucmd
.sbuf_size
, 0, 0);
267 if (IS_ERR(qp
->sumem
)) {
268 ib_umem_release(qp
->rumem
);
269 ret
= PTR_ERR(qp
->sumem
);
273 qp
->npages_send
= ib_umem_page_count(qp
->sumem
);
274 qp
->npages_recv
= ib_umem_page_count(qp
->rumem
);
275 qp
->npages
= qp
->npages_send
+ qp
->npages_recv
;
277 qp
->is_kernel
= true;
279 ret
= pvrdma_set_sq_size(to_vdev(pd
->device
),
280 &init_attr
->cap
, qp
);
284 ret
= pvrdma_set_rq_size(to_vdev(pd
->device
),
285 &init_attr
->cap
, qp
);
289 qp
->npages
= qp
->npages_send
+ qp
->npages_recv
;
291 /* Skip header page. */
292 qp
->sq
.offset
= PVRDMA_QP_NUM_HEADER_PAGES
* PAGE_SIZE
;
294 /* Recv queue pages are after send pages. */
295 qp
->rq
.offset
= qp
->npages_send
* PAGE_SIZE
;
298 if (qp
->npages
< 0 || qp
->npages
> PVRDMA_PAGE_DIR_MAX_PAGES
) {
299 dev_warn(&dev
->pdev
->dev
,
300 "overflow pages in queuepair\n");
305 ret
= pvrdma_page_dir_init(dev
, &qp
->pdir
, qp
->npages
,
308 dev_warn(&dev
->pdev
->dev
,
309 "could not allocate page directory\n");
313 if (!qp
->is_kernel
) {
314 pvrdma_page_dir_insert_umem(&qp
->pdir
, qp
->sumem
, 0);
315 pvrdma_page_dir_insert_umem(&qp
->pdir
, qp
->rumem
,
318 /* Ring state is always the first page. */
319 qp
->sq
.ring
= qp
->pdir
.pages
[0];
320 qp
->rq
.ring
= &qp
->sq
.ring
[1];
329 init_attr
->cap
.max_inline_data
= 0;
331 memset(cmd
, 0, sizeof(*cmd
));
332 cmd
->hdr
.cmd
= PVRDMA_CMD_CREATE_QP
;
333 cmd
->pd_handle
= to_vpd(pd
)->pd_handle
;
334 cmd
->send_cq_handle
= to_vcq(init_attr
->send_cq
)->cq_handle
;
335 cmd
->recv_cq_handle
= to_vcq(init_attr
->recv_cq
)->cq_handle
;
336 cmd
->max_send_wr
= init_attr
->cap
.max_send_wr
;
337 cmd
->max_recv_wr
= init_attr
->cap
.max_recv_wr
;
338 cmd
->max_send_sge
= init_attr
->cap
.max_send_sge
;
339 cmd
->max_recv_sge
= init_attr
->cap
.max_recv_sge
;
340 cmd
->max_inline_data
= init_attr
->cap
.max_inline_data
;
341 cmd
->sq_sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
) ? 1 : 0;
342 cmd
->qp_type
= ib_qp_type_to_pvrdma(init_attr
->qp_type
);
343 cmd
->access_flags
= IB_ACCESS_LOCAL_WRITE
;
344 cmd
->total_chunks
= qp
->npages
;
345 cmd
->send_chunks
= qp
->npages_send
- PVRDMA_QP_NUM_HEADER_PAGES
;
346 cmd
->pdir_dma
= qp
->pdir
.dir_dma
;
348 dev_dbg(&dev
->pdev
->dev
, "create queuepair with %d, %d, %d, %d\n",
349 cmd
->max_send_wr
, cmd
->max_recv_wr
, cmd
->max_send_sge
,
352 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_CREATE_QP_RESP
);
354 dev_warn(&dev
->pdev
->dev
,
355 "could not create queuepair, error: %d\n", ret
);
359 /* max_send_wr/_recv_wr/_send_sge/_recv_sge/_inline_data */
360 qp
->qp_handle
= resp
->qpn
;
361 qp
->port
= init_attr
->port_num
;
362 qp
->ibqp
.qp_num
= resp
->qpn
;
363 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
364 dev
->qp_tbl
[qp
->qp_handle
% dev
->dsr
->caps
.max_qp
] = qp
;
365 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
370 pvrdma_page_dir_cleanup(dev
, &qp
->pdir
);
372 if (pd
->uobject
&& udata
) {
374 ib_umem_release(qp
->rumem
);
376 ib_umem_release(qp
->sumem
);
380 atomic_dec(&dev
->num_qps
);
385 static void pvrdma_free_qp(struct pvrdma_qp
*qp
)
387 struct pvrdma_dev
*dev
= to_vdev(qp
->ibqp
.device
);
388 struct pvrdma_cq
*scq
;
389 struct pvrdma_cq
*rcq
;
390 unsigned long flags
, scq_flags
, rcq_flags
;
392 /* In case cq is polling */
393 get_cqs(qp
, &scq
, &rcq
);
394 pvrdma_lock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
396 _pvrdma_flush_cqe(qp
, scq
);
398 _pvrdma_flush_cqe(qp
, rcq
);
400 spin_lock_irqsave(&dev
->qp_tbl_lock
, flags
);
401 dev
->qp_tbl
[qp
->qp_handle
] = NULL
;
402 spin_unlock_irqrestore(&dev
->qp_tbl_lock
, flags
);
404 pvrdma_unlock_cqs(scq
, rcq
, &scq_flags
, &rcq_flags
);
406 atomic_dec(&qp
->refcnt
);
407 wait_event(qp
->wait
, !atomic_read(&qp
->refcnt
));
409 pvrdma_page_dir_cleanup(dev
, &qp
->pdir
);
413 atomic_dec(&dev
->num_qps
);
417 * pvrdma_destroy_qp - destroy a queue pair
418 * @qp: the queue pair to destroy
420 * @return: 0 on success.
422 int pvrdma_destroy_qp(struct ib_qp
*qp
)
424 struct pvrdma_qp
*vqp
= to_vqp(qp
);
425 union pvrdma_cmd_req req
;
426 struct pvrdma_cmd_destroy_qp
*cmd
= &req
.destroy_qp
;
429 memset(cmd
, 0, sizeof(*cmd
));
430 cmd
->hdr
.cmd
= PVRDMA_CMD_DESTROY_QP
;
431 cmd
->qp_handle
= vqp
->qp_handle
;
433 ret
= pvrdma_cmd_post(to_vdev(qp
->device
), &req
, NULL
, 0);
435 dev_warn(&to_vdev(qp
->device
)->pdev
->dev
,
436 "destroy queuepair failed, error: %d\n", ret
);
444 * pvrdma_modify_qp - modify queue pair attributes
445 * @ibqp: the queue pair
446 * @attr: the new queue pair's attributes
447 * @attr_mask: attributes mask
450 * @returns 0 on success, otherwise returns an errno.
452 int pvrdma_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
453 int attr_mask
, struct ib_udata
*udata
)
455 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
456 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
457 union pvrdma_cmd_req req
;
458 union pvrdma_cmd_resp rsp
;
459 struct pvrdma_cmd_modify_qp
*cmd
= &req
.modify_qp
;
460 int cur_state
, next_state
;
463 /* Sanity checking. Should need lock here */
464 mutex_lock(&qp
->mutex
);
465 cur_state
= (attr_mask
& IB_QP_CUR_STATE
) ? attr
->cur_qp_state
:
467 next_state
= (attr_mask
& IB_QP_STATE
) ? attr
->qp_state
: cur_state
;
469 if (!ib_modify_qp_is_ok(cur_state
, next_state
, ibqp
->qp_type
,
470 attr_mask
, IB_LINK_LAYER_ETHERNET
)) {
475 if (attr_mask
& IB_QP_PORT
) {
476 if (attr
->port_num
== 0 ||
477 attr
->port_num
> ibqp
->device
->phys_port_cnt
) {
483 if (attr_mask
& IB_QP_MIN_RNR_TIMER
) {
484 if (attr
->min_rnr_timer
> 31) {
490 if (attr_mask
& IB_QP_PKEY_INDEX
) {
491 if (attr
->pkey_index
>= dev
->dsr
->caps
.max_pkeys
) {
497 if (attr_mask
& IB_QP_QKEY
)
498 qp
->qkey
= attr
->qkey
;
500 if (cur_state
== next_state
&& cur_state
== IB_QPS_RESET
) {
505 qp
->state
= next_state
;
506 memset(cmd
, 0, sizeof(*cmd
));
507 cmd
->hdr
.cmd
= PVRDMA_CMD_MODIFY_QP
;
508 cmd
->qp_handle
= qp
->qp_handle
;
509 cmd
->attr_mask
= ib_qp_attr_mask_to_pvrdma(attr_mask
);
510 cmd
->attrs
.qp_state
= ib_qp_state_to_pvrdma(attr
->qp_state
);
511 cmd
->attrs
.cur_qp_state
=
512 ib_qp_state_to_pvrdma(attr
->cur_qp_state
);
513 cmd
->attrs
.path_mtu
= ib_mtu_to_pvrdma(attr
->path_mtu
);
514 cmd
->attrs
.path_mig_state
=
515 ib_mig_state_to_pvrdma(attr
->path_mig_state
);
516 cmd
->attrs
.qkey
= attr
->qkey
;
517 cmd
->attrs
.rq_psn
= attr
->rq_psn
;
518 cmd
->attrs
.sq_psn
= attr
->sq_psn
;
519 cmd
->attrs
.dest_qp_num
= attr
->dest_qp_num
;
520 cmd
->attrs
.qp_access_flags
=
521 ib_access_flags_to_pvrdma(attr
->qp_access_flags
);
522 cmd
->attrs
.pkey_index
= attr
->pkey_index
;
523 cmd
->attrs
.alt_pkey_index
= attr
->alt_pkey_index
;
524 cmd
->attrs
.en_sqd_async_notify
= attr
->en_sqd_async_notify
;
525 cmd
->attrs
.sq_draining
= attr
->sq_draining
;
526 cmd
->attrs
.max_rd_atomic
= attr
->max_rd_atomic
;
527 cmd
->attrs
.max_dest_rd_atomic
= attr
->max_dest_rd_atomic
;
528 cmd
->attrs
.min_rnr_timer
= attr
->min_rnr_timer
;
529 cmd
->attrs
.port_num
= attr
->port_num
;
530 cmd
->attrs
.timeout
= attr
->timeout
;
531 cmd
->attrs
.retry_cnt
= attr
->retry_cnt
;
532 cmd
->attrs
.rnr_retry
= attr
->rnr_retry
;
533 cmd
->attrs
.alt_port_num
= attr
->alt_port_num
;
534 cmd
->attrs
.alt_timeout
= attr
->alt_timeout
;
535 ib_qp_cap_to_pvrdma(&cmd
->attrs
.cap
, &attr
->cap
);
536 rdma_ah_attr_to_pvrdma(&cmd
->attrs
.ah_attr
, &attr
->ah_attr
);
537 rdma_ah_attr_to_pvrdma(&cmd
->attrs
.alt_ah_attr
, &attr
->alt_ah_attr
);
539 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_MODIFY_QP_RESP
);
541 dev_warn(&dev
->pdev
->dev
,
542 "could not modify queuepair, error: %d\n", ret
);
543 } else if (rsp
.hdr
.err
> 0) {
544 dev_warn(&dev
->pdev
->dev
,
545 "cannot modify queuepair, error: %d\n", rsp
.hdr
.err
);
549 if (ret
== 0 && next_state
== IB_QPS_RESET
)
553 mutex_unlock(&qp
->mutex
);
558 static inline void *get_sq_wqe(struct pvrdma_qp
*qp
, unsigned int n
)
560 return pvrdma_page_dir_get_ptr(&qp
->pdir
,
561 qp
->sq
.offset
+ n
* qp
->sq
.wqe_size
);
564 static inline void *get_rq_wqe(struct pvrdma_qp
*qp
, unsigned int n
)
566 return pvrdma_page_dir_get_ptr(&qp
->pdir
,
567 qp
->rq
.offset
+ n
* qp
->rq
.wqe_size
);
570 static int set_reg_seg(struct pvrdma_sq_wqe_hdr
*wqe_hdr
, struct ib_reg_wr
*wr
)
572 struct pvrdma_user_mr
*mr
= to_vmr(wr
->mr
);
574 wqe_hdr
->wr
.fast_reg
.iova_start
= mr
->ibmr
.iova
;
575 wqe_hdr
->wr
.fast_reg
.pl_pdir_dma
= mr
->pdir
.dir_dma
;
576 wqe_hdr
->wr
.fast_reg
.page_shift
= mr
->page_shift
;
577 wqe_hdr
->wr
.fast_reg
.page_list_len
= mr
->npages
;
578 wqe_hdr
->wr
.fast_reg
.length
= mr
->ibmr
.length
;
579 wqe_hdr
->wr
.fast_reg
.access_flags
= wr
->access
;
580 wqe_hdr
->wr
.fast_reg
.rkey
= wr
->key
;
582 return pvrdma_page_dir_insert_page_list(&mr
->pdir
, mr
->pages
,
587 * pvrdma_post_send - post send work request entries on a QP
589 * @wr: work request list to post
590 * @bad_wr: the first bad WR returned
592 * @return: 0 on success, otherwise errno returned.
594 int pvrdma_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*wr
,
595 struct ib_send_wr
**bad_wr
)
597 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
598 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
600 struct pvrdma_sq_wqe_hdr
*wqe_hdr
;
601 struct pvrdma_sge
*sge
;
605 * In states lower than RTS, we can fail immediately. In other states,
606 * just post and let the device figure it out.
608 if (qp
->state
< IB_QPS_RTS
) {
613 spin_lock_irqsave(&qp
->sq
.lock
, flags
);
616 unsigned int tail
= 0;
618 if (unlikely(!pvrdma_idx_ring_has_space(
619 qp
->sq
.ring
, qp
->sq
.wqe_cnt
, &tail
))) {
620 dev_warn_ratelimited(&dev
->pdev
->dev
,
621 "send queue is full\n");
627 if (unlikely(wr
->num_sge
> qp
->sq
.max_sg
|| wr
->num_sge
< 0)) {
628 dev_warn_ratelimited(&dev
->pdev
->dev
,
629 "send SGE overflow\n");
635 if (unlikely(wr
->opcode
< 0)) {
636 dev_warn_ratelimited(&dev
->pdev
->dev
,
637 "invalid send opcode\n");
644 * Only support UD, RC.
645 * Need to check opcode table for thorough checking.
648 * _SEND_WITH_IMM x x x
650 * _RDMA_WRITE_WITH_IMM x x
654 * _ATOMIC_CMP_AND_SWP x
655 * _ATOMIC_FETCH_AND_ADD x
656 * _MASK_ATOMIC_CMP_AND_SWP x
657 * _MASK_ATOMIC_FETCH_AND_ADD x
661 if (qp
->ibqp
.qp_type
!= IB_QPT_UD
&&
662 qp
->ibqp
.qp_type
!= IB_QPT_RC
&&
663 wr
->opcode
!= IB_WR_SEND
) {
664 dev_warn_ratelimited(&dev
->pdev
->dev
,
665 "unsupported queuepair type\n");
669 } else if (qp
->ibqp
.qp_type
== IB_QPT_UD
||
670 qp
->ibqp
.qp_type
== IB_QPT_GSI
) {
671 if (wr
->opcode
!= IB_WR_SEND
&&
672 wr
->opcode
!= IB_WR_SEND_WITH_IMM
) {
673 dev_warn_ratelimited(&dev
->pdev
->dev
,
674 "invalid send opcode\n");
681 wqe_hdr
= (struct pvrdma_sq_wqe_hdr
*)get_sq_wqe(qp
, tail
);
682 memset(wqe_hdr
, 0, sizeof(*wqe_hdr
));
683 wqe_hdr
->wr_id
= wr
->wr_id
;
684 wqe_hdr
->num_sge
= wr
->num_sge
;
685 wqe_hdr
->opcode
= ib_wr_opcode_to_pvrdma(wr
->opcode
);
686 wqe_hdr
->send_flags
= ib_send_flags_to_pvrdma(wr
->send_flags
);
687 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
||
688 wr
->opcode
== IB_WR_RDMA_WRITE_WITH_IMM
)
689 wqe_hdr
->ex
.imm_data
= wr
->ex
.imm_data
;
691 switch (qp
->ibqp
.qp_type
) {
694 if (unlikely(!ud_wr(wr
)->ah
)) {
695 dev_warn_ratelimited(&dev
->pdev
->dev
,
696 "invalid address handle\n");
703 * Use qkey from qp context if high order bit set,
704 * otherwise from work request.
706 wqe_hdr
->wr
.ud
.remote_qpn
= ud_wr(wr
)->remote_qpn
;
707 wqe_hdr
->wr
.ud
.remote_qkey
=
708 ud_wr(wr
)->remote_qkey
& 0x80000000 ?
709 qp
->qkey
: ud_wr(wr
)->remote_qkey
;
710 wqe_hdr
->wr
.ud
.av
= to_vah(ud_wr(wr
)->ah
)->av
;
714 switch (wr
->opcode
) {
715 case IB_WR_RDMA_READ
:
716 case IB_WR_RDMA_WRITE
:
717 case IB_WR_RDMA_WRITE_WITH_IMM
:
718 wqe_hdr
->wr
.rdma
.remote_addr
=
719 rdma_wr(wr
)->remote_addr
;
720 wqe_hdr
->wr
.rdma
.rkey
= rdma_wr(wr
)->rkey
;
722 case IB_WR_LOCAL_INV
:
723 case IB_WR_SEND_WITH_INV
:
724 wqe_hdr
->ex
.invalidate_rkey
=
725 wr
->ex
.invalidate_rkey
;
727 case IB_WR_ATOMIC_CMP_AND_SWP
:
728 case IB_WR_ATOMIC_FETCH_AND_ADD
:
729 wqe_hdr
->wr
.atomic
.remote_addr
=
730 atomic_wr(wr
)->remote_addr
;
731 wqe_hdr
->wr
.atomic
.rkey
= atomic_wr(wr
)->rkey
;
732 wqe_hdr
->wr
.atomic
.compare_add
=
733 atomic_wr(wr
)->compare_add
;
734 if (wr
->opcode
== IB_WR_ATOMIC_CMP_AND_SWP
)
735 wqe_hdr
->wr
.atomic
.swap
=
739 ret
= set_reg_seg(wqe_hdr
, reg_wr(wr
));
741 dev_warn_ratelimited(&dev
->pdev
->dev
,
742 "Failed to set fast register work request\n");
753 dev_warn_ratelimited(&dev
->pdev
->dev
,
754 "invalid queuepair type\n");
760 sge
= (struct pvrdma_sge
*)(wqe_hdr
+ 1);
761 for (i
= 0; i
< wr
->num_sge
; i
++) {
762 /* Need to check wqe_size 0 or max size */
763 sge
->addr
= wr
->sg_list
[i
].addr
;
764 sge
->length
= wr
->sg_list
[i
].length
;
765 sge
->lkey
= wr
->sg_list
[i
].lkey
;
769 /* Make sure wqe is written before index update */
772 /* Update shared sq ring */
773 pvrdma_idx_ring_inc(&qp
->sq
.ring
->prod_tail
,
782 spin_unlock_irqrestore(&qp
->sq
.lock
, flags
);
785 pvrdma_write_uar_qp(dev
, PVRDMA_UAR_QP_SEND
| qp
->qp_handle
);
791 * pvrdma_post_receive - post receive work request entries on a QP
793 * @wr: the work request list to post
794 * @bad_wr: the first bad WR returned
796 * @return: 0 on success, otherwise errno returned.
798 int pvrdma_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*wr
,
799 struct ib_recv_wr
**bad_wr
)
801 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
803 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
804 struct pvrdma_rq_wqe_hdr
*wqe_hdr
;
805 struct pvrdma_sge
*sge
;
810 * In the RESET state, we can fail immediately. For other states,
811 * just post and let the device figure it out.
813 if (qp
->state
== IB_QPS_RESET
) {
818 spin_lock_irqsave(&qp
->rq
.lock
, flags
);
821 unsigned int tail
= 0;
823 if (unlikely(wr
->num_sge
> qp
->rq
.max_sg
||
827 dev_warn_ratelimited(&dev
->pdev
->dev
,
828 "recv SGE overflow\n");
832 if (unlikely(!pvrdma_idx_ring_has_space(
833 qp
->rq
.ring
, qp
->rq
.wqe_cnt
, &tail
))) {
836 dev_warn_ratelimited(&dev
->pdev
->dev
,
837 "recv queue full\n");
841 wqe_hdr
= (struct pvrdma_rq_wqe_hdr
*)get_rq_wqe(qp
, tail
);
842 wqe_hdr
->wr_id
= wr
->wr_id
;
843 wqe_hdr
->num_sge
= wr
->num_sge
;
844 wqe_hdr
->total_len
= 0;
846 sge
= (struct pvrdma_sge
*)(wqe_hdr
+ 1);
847 for (i
= 0; i
< wr
->num_sge
; i
++) {
848 sge
->addr
= wr
->sg_list
[i
].addr
;
849 sge
->length
= wr
->sg_list
[i
].length
;
850 sge
->lkey
= wr
->sg_list
[i
].lkey
;
854 /* Make sure wqe is written before index update */
857 /* Update shared rq ring */
858 pvrdma_idx_ring_inc(&qp
->rq
.ring
->prod_tail
,
864 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
866 pvrdma_write_uar_qp(dev
, PVRDMA_UAR_QP_RECV
| qp
->qp_handle
);
871 spin_unlock_irqrestore(&qp
->rq
.lock
, flags
);
877 * pvrdma_query_qp - query a queue pair's attributes
878 * @ibqp: the queue pair to query
879 * @attr: the queue pair's attributes
880 * @attr_mask: attributes mask
881 * @init_attr: initial queue pair attributes
883 * @returns 0 on success, otherwise returns an errno.
885 int pvrdma_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
886 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
888 struct pvrdma_dev
*dev
= to_vdev(ibqp
->device
);
889 struct pvrdma_qp
*qp
= to_vqp(ibqp
);
890 union pvrdma_cmd_req req
;
891 union pvrdma_cmd_resp rsp
;
892 struct pvrdma_cmd_query_qp
*cmd
= &req
.query_qp
;
893 struct pvrdma_cmd_query_qp_resp
*resp
= &rsp
.query_qp_resp
;
896 mutex_lock(&qp
->mutex
);
898 if (qp
->state
== IB_QPS_RESET
) {
899 attr
->qp_state
= IB_QPS_RESET
;
903 memset(cmd
, 0, sizeof(*cmd
));
904 cmd
->hdr
.cmd
= PVRDMA_CMD_QUERY_QP
;
905 cmd
->qp_handle
= qp
->qp_handle
;
906 cmd
->attr_mask
= ib_qp_attr_mask_to_pvrdma(attr_mask
);
908 ret
= pvrdma_cmd_post(dev
, &req
, &rsp
, PVRDMA_CMD_QUERY_QP_RESP
);
910 dev_warn(&dev
->pdev
->dev
,
911 "could not query queuepair, error: %d\n", ret
);
915 attr
->qp_state
= pvrdma_qp_state_to_ib(resp
->attrs
.qp_state
);
917 pvrdma_qp_state_to_ib(resp
->attrs
.cur_qp_state
);
918 attr
->path_mtu
= pvrdma_mtu_to_ib(resp
->attrs
.path_mtu
);
919 attr
->path_mig_state
=
920 pvrdma_mig_state_to_ib(resp
->attrs
.path_mig_state
);
921 attr
->qkey
= resp
->attrs
.qkey
;
922 attr
->rq_psn
= resp
->attrs
.rq_psn
;
923 attr
->sq_psn
= resp
->attrs
.sq_psn
;
924 attr
->dest_qp_num
= resp
->attrs
.dest_qp_num
;
925 attr
->qp_access_flags
=
926 pvrdma_access_flags_to_ib(resp
->attrs
.qp_access_flags
);
927 attr
->pkey_index
= resp
->attrs
.pkey_index
;
928 attr
->alt_pkey_index
= resp
->attrs
.alt_pkey_index
;
929 attr
->en_sqd_async_notify
= resp
->attrs
.en_sqd_async_notify
;
930 attr
->sq_draining
= resp
->attrs
.sq_draining
;
931 attr
->max_rd_atomic
= resp
->attrs
.max_rd_atomic
;
932 attr
->max_dest_rd_atomic
= resp
->attrs
.max_dest_rd_atomic
;
933 attr
->min_rnr_timer
= resp
->attrs
.min_rnr_timer
;
934 attr
->port_num
= resp
->attrs
.port_num
;
935 attr
->timeout
= resp
->attrs
.timeout
;
936 attr
->retry_cnt
= resp
->attrs
.retry_cnt
;
937 attr
->rnr_retry
= resp
->attrs
.rnr_retry
;
938 attr
->alt_port_num
= resp
->attrs
.alt_port_num
;
939 attr
->alt_timeout
= resp
->attrs
.alt_timeout
;
940 pvrdma_qp_cap_to_ib(&attr
->cap
, &resp
->attrs
.cap
);
941 pvrdma_ah_attr_to_rdma(&attr
->ah_attr
, &resp
->attrs
.ah_attr
);
942 pvrdma_ah_attr_to_rdma(&attr
->alt_ah_attr
, &resp
->attrs
.alt_ah_attr
);
944 qp
->state
= attr
->qp_state
;
949 attr
->cur_qp_state
= attr
->qp_state
;
951 init_attr
->event_handler
= qp
->ibqp
.event_handler
;
952 init_attr
->qp_context
= qp
->ibqp
.qp_context
;
953 init_attr
->send_cq
= qp
->ibqp
.send_cq
;
954 init_attr
->recv_cq
= qp
->ibqp
.recv_cq
;
955 init_attr
->srq
= qp
->ibqp
.srq
;
956 init_attr
->xrcd
= NULL
;
957 init_attr
->cap
= attr
->cap
;
958 init_attr
->sq_sig_type
= 0;
959 init_attr
->qp_type
= qp
->ibqp
.qp_type
;
960 init_attr
->create_flags
= 0;
961 init_attr
->port_num
= qp
->port
;
963 mutex_unlock(&qp
->mutex
);