2 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/mlx4/qp.h>
35 #include <linux/mlx4/srq.h>
40 static void *get_wqe(struct mlx4_ib_srq
*srq
, int n
)
42 return mlx4_buf_offset(&srq
->buf
, n
<< srq
->msrq
.wqe_shift
);
45 static void mlx4_ib_srq_event(struct mlx4_srq
*srq
, enum mlx4_event type
)
47 struct ib_event event
;
48 struct ib_srq
*ibsrq
= &to_mibsrq(srq
)->ibsrq
;
50 if (ibsrq
->event_handler
) {
51 event
.device
= ibsrq
->device
;
52 event
.element
.srq
= ibsrq
;
54 case MLX4_EVENT_TYPE_SRQ_LIMIT
:
55 event
.event
= IB_EVENT_SRQ_LIMIT_REACHED
;
57 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR
:
58 event
.event
= IB_EVENT_SRQ_ERR
;
61 printk(KERN_WARNING
"mlx4_ib: Unexpected event type %d "
62 "on SRQ %06x\n", type
, srq
->srqn
);
66 ibsrq
->event_handler(&event
, ibsrq
->srq_context
);
70 struct ib_srq
*mlx4_ib_create_srq(struct ib_pd
*pd
,
71 struct ib_srq_init_attr
*init_attr
,
72 struct ib_udata
*udata
)
74 struct mlx4_ib_dev
*dev
= to_mdev(pd
->device
);
75 struct mlx4_ib_srq
*srq
;
76 struct mlx4_wqe_srq_next_seg
*next
;
77 struct mlx4_wqe_data_seg
*scatter
;
83 /* Sanity check SRQ size before proceeding */
84 if (init_attr
->attr
.max_wr
>= dev
->dev
->caps
.max_srq_wqes
||
85 init_attr
->attr
.max_sge
> dev
->dev
->caps
.max_srq_sge
)
86 return ERR_PTR(-EINVAL
);
88 srq
= kmalloc(sizeof *srq
, GFP_KERNEL
);
90 return ERR_PTR(-ENOMEM
);
92 mutex_init(&srq
->mutex
);
93 spin_lock_init(&srq
->lock
);
94 srq
->msrq
.max
= roundup_pow_of_two(init_attr
->attr
.max_wr
+ 1);
95 srq
->msrq
.max_gs
= init_attr
->attr
.max_sge
;
98 roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg
) +
100 sizeof (struct mlx4_wqe_data_seg
)));
101 srq
->msrq
.wqe_shift
= ilog2(desc_size
);
103 buf_size
= srq
->msrq
.max
* desc_size
;
106 struct mlx4_ib_create_srq ucmd
;
108 if (ib_copy_from_udata(&ucmd
, udata
, sizeof ucmd
)) {
113 srq
->umem
= ib_umem_get(pd
->uobject
->context
, ucmd
.buf_addr
,
115 if (IS_ERR(srq
->umem
)) {
116 err
= PTR_ERR(srq
->umem
);
120 err
= mlx4_mtt_init(dev
->dev
, ib_umem_page_count(srq
->umem
),
121 ilog2(srq
->umem
->page_size
), &srq
->mtt
);
125 err
= mlx4_ib_umem_write_mtt(dev
, &srq
->mtt
, srq
->umem
);
129 err
= mlx4_ib_db_map_user(to_mucontext(pd
->uobject
->context
),
130 ucmd
.db_addr
, &srq
->db
);
134 err
= mlx4_db_alloc(dev
->dev
, &srq
->db
, 0);
140 if (mlx4_buf_alloc(dev
->dev
, buf_size
, PAGE_SIZE
* 2, &srq
->buf
)) {
146 srq
->tail
= srq
->msrq
.max
- 1;
149 for (i
= 0; i
< srq
->msrq
.max
; ++i
) {
150 next
= get_wqe(srq
, i
);
151 next
->next_wqe_index
=
152 cpu_to_be16((i
+ 1) & (srq
->msrq
.max
- 1));
154 for (scatter
= (void *) (next
+ 1);
155 (void *) scatter
< (void *) next
+ desc_size
;
157 scatter
->lkey
= cpu_to_be32(MLX4_INVALID_LKEY
);
160 err
= mlx4_mtt_init(dev
->dev
, srq
->buf
.npages
, srq
->buf
.page_shift
,
165 err
= mlx4_buf_write_mtt(dev
->dev
, &srq
->mtt
, &srq
->buf
);
169 srq
->wrid
= kmalloc(srq
->msrq
.max
* sizeof (u64
), GFP_KERNEL
);
176 err
= mlx4_srq_alloc(dev
->dev
, to_mpd(pd
)->pdn
, &srq
->mtt
,
177 srq
->db
.dma
, &srq
->msrq
);
181 srq
->msrq
.event
= mlx4_ib_srq_event
;
184 if (ib_copy_to_udata(udata
, &srq
->msrq
.srqn
, sizeof (__u32
))) {
189 init_attr
->attr
.max_wr
= srq
->msrq
.max
- 1;
195 mlx4_ib_db_unmap_user(to_mucontext(pd
->uobject
->context
), &srq
->db
);
200 mlx4_mtt_cleanup(dev
->dev
, &srq
->mtt
);
204 ib_umem_release(srq
->umem
);
206 mlx4_buf_free(dev
->dev
, buf_size
, &srq
->buf
);
210 mlx4_db_free(dev
->dev
, &srq
->db
);
218 int mlx4_ib_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
219 enum ib_srq_attr_mask attr_mask
, struct ib_udata
*udata
)
221 struct mlx4_ib_dev
*dev
= to_mdev(ibsrq
->device
);
222 struct mlx4_ib_srq
*srq
= to_msrq(ibsrq
);
225 /* We don't support resizing SRQs (yet?) */
226 if (attr_mask
& IB_SRQ_MAX_WR
)
229 if (attr_mask
& IB_SRQ_LIMIT
) {
230 if (attr
->srq_limit
>= srq
->msrq
.max
)
233 mutex_lock(&srq
->mutex
);
234 ret
= mlx4_srq_arm(dev
->dev
, &srq
->msrq
, attr
->srq_limit
);
235 mutex_unlock(&srq
->mutex
);
244 int mlx4_ib_query_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*srq_attr
)
246 struct mlx4_ib_dev
*dev
= to_mdev(ibsrq
->device
);
247 struct mlx4_ib_srq
*srq
= to_msrq(ibsrq
);
251 ret
= mlx4_srq_query(dev
->dev
, &srq
->msrq
, &limit_watermark
);
255 srq_attr
->srq_limit
= limit_watermark
;
256 srq_attr
->max_wr
= srq
->msrq
.max
- 1;
257 srq_attr
->max_sge
= srq
->msrq
.max_gs
;
262 int mlx4_ib_destroy_srq(struct ib_srq
*srq
)
264 struct mlx4_ib_dev
*dev
= to_mdev(srq
->device
);
265 struct mlx4_ib_srq
*msrq
= to_msrq(srq
);
267 mlx4_srq_free(dev
->dev
, &msrq
->msrq
);
268 mlx4_mtt_cleanup(dev
->dev
, &msrq
->mtt
);
271 mlx4_ib_db_unmap_user(to_mucontext(srq
->uobject
->context
), &msrq
->db
);
272 ib_umem_release(msrq
->umem
);
275 mlx4_buf_free(dev
->dev
, msrq
->msrq
.max
<< msrq
->msrq
.wqe_shift
,
277 mlx4_db_free(dev
->dev
, &msrq
->db
);
285 void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq
*srq
, int wqe_index
)
287 struct mlx4_wqe_srq_next_seg
*next
;
289 /* always called with interrupts disabled. */
290 spin_lock(&srq
->lock
);
292 next
= get_wqe(srq
, srq
->tail
);
293 next
->next_wqe_index
= cpu_to_be16(wqe_index
);
294 srq
->tail
= wqe_index
;
296 spin_unlock(&srq
->lock
);
299 int mlx4_ib_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
300 struct ib_recv_wr
**bad_wr
)
302 struct mlx4_ib_srq
*srq
= to_msrq(ibsrq
);
303 struct mlx4_wqe_srq_next_seg
*next
;
304 struct mlx4_wqe_data_seg
*scat
;
310 spin_lock_irqsave(&srq
->lock
, flags
);
312 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
313 if (unlikely(wr
->num_sge
> srq
->msrq
.max_gs
)) {
319 if (unlikely(srq
->head
== srq
->tail
)) {
325 srq
->wrid
[srq
->head
] = wr
->wr_id
;
327 next
= get_wqe(srq
, srq
->head
);
328 srq
->head
= be16_to_cpu(next
->next_wqe_index
);
329 scat
= (struct mlx4_wqe_data_seg
*) (next
+ 1);
331 for (i
= 0; i
< wr
->num_sge
; ++i
) {
332 scat
[i
].byte_count
= cpu_to_be32(wr
->sg_list
[i
].length
);
333 scat
[i
].lkey
= cpu_to_be32(wr
->sg_list
[i
].lkey
);
334 scat
[i
].addr
= cpu_to_be64(wr
->sg_list
[i
].addr
);
337 if (i
< srq
->msrq
.max_gs
) {
338 scat
[i
].byte_count
= 0;
339 scat
[i
].lkey
= cpu_to_be32(MLX4_INVALID_LKEY
);
345 srq
->wqe_ctr
+= nreq
;
348 * Make sure that descriptors are written before
353 *srq
->db
.db
= cpu_to_be32(srq
->wqe_ctr
);
356 spin_unlock_irqrestore(&srq
->lock
, flags
);