2 * Broadcom NetXtreme-E RoCE driver.
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Description: IB Verbs interpreter
39 #include <linux/interrupt.h>
40 #include <linux/types.h>
41 #include <linux/pci.h>
42 #include <linux/netdevice.h>
43 #include <linux/if_ether.h>
45 #include <rdma/ib_verbs.h>
46 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
48 #include <rdma/ib_addr.h>
49 #include <rdma/ib_mad.h>
50 #include <rdma/ib_cache.h>
55 #include "qplib_res.h"
58 #include "qplib_rcfw.h"
62 #include <rdma/bnxt_re-abi.h>
64 static int bnxt_re_build_sgl(struct ib_sge
*ib_sg_list
,
65 struct bnxt_qplib_sge
*sg_list
, int num
)
69 for (i
= 0; i
< num
; i
++) {
70 sg_list
[i
].addr
= ib_sg_list
[i
].addr
;
71 sg_list
[i
].lkey
= ib_sg_list
[i
].lkey
;
72 sg_list
[i
].size
= ib_sg_list
[i
].length
;
73 total
+= sg_list
[i
].size
;
79 struct net_device
*bnxt_re_get_netdev(struct ib_device
*ibdev
, u8 port_num
)
81 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
82 struct net_device
*netdev
= NULL
;
86 netdev
= rdev
->netdev
;
94 int bnxt_re_query_device(struct ib_device
*ibdev
,
95 struct ib_device_attr
*ib_attr
,
96 struct ib_udata
*udata
)
98 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
99 struct bnxt_qplib_dev_attr
*dev_attr
= &rdev
->dev_attr
;
101 memset(ib_attr
, 0, sizeof(*ib_attr
));
103 ib_attr
->fw_ver
= (u64
)(unsigned long)(dev_attr
->fw_ver
);
104 bnxt_qplib_get_guid(rdev
->netdev
->dev_addr
,
105 (u8
*)&ib_attr
->sys_image_guid
);
106 ib_attr
->max_mr_size
= ~0ull;
107 ib_attr
->page_size_cap
= BNXT_RE_PAGE_SIZE_4K
| BNXT_RE_PAGE_SIZE_8K
|
108 BNXT_RE_PAGE_SIZE_64K
| BNXT_RE_PAGE_SIZE_2M
|
109 BNXT_RE_PAGE_SIZE_8M
| BNXT_RE_PAGE_SIZE_1G
;
111 ib_attr
->vendor_id
= rdev
->en_dev
->pdev
->vendor
;
112 ib_attr
->vendor_part_id
= rdev
->en_dev
->pdev
->device
;
113 ib_attr
->hw_ver
= rdev
->en_dev
->pdev
->subsystem_device
;
114 ib_attr
->max_qp
= dev_attr
->max_qp
;
115 ib_attr
->max_qp_wr
= dev_attr
->max_qp_wqes
;
116 ib_attr
->device_cap_flags
=
117 IB_DEVICE_CURR_QP_STATE_MOD
118 | IB_DEVICE_RC_RNR_NAK_GEN
119 | IB_DEVICE_SHUTDOWN_PORT
120 | IB_DEVICE_SYS_IMAGE_GUID
121 | IB_DEVICE_LOCAL_DMA_LKEY
122 | IB_DEVICE_RESIZE_MAX_WR
123 | IB_DEVICE_PORT_ACTIVE_EVENT
124 | IB_DEVICE_N_NOTIFY_CQ
125 | IB_DEVICE_MEM_WINDOW
126 | IB_DEVICE_MEM_WINDOW_TYPE_2B
127 | IB_DEVICE_MEM_MGT_EXTENSIONS
;
128 ib_attr
->max_sge
= dev_attr
->max_qp_sges
;
129 ib_attr
->max_sge_rd
= dev_attr
->max_qp_sges
;
130 ib_attr
->max_cq
= dev_attr
->max_cq
;
131 ib_attr
->max_cqe
= dev_attr
->max_cq_wqes
;
132 ib_attr
->max_mr
= dev_attr
->max_mr
;
133 ib_attr
->max_pd
= dev_attr
->max_pd
;
134 ib_attr
->max_qp_rd_atom
= dev_attr
->max_qp_rd_atom
;
135 ib_attr
->max_qp_init_rd_atom
= dev_attr
->max_qp_rd_atom
;
136 ib_attr
->atomic_cap
= IB_ATOMIC_HCA
;
137 ib_attr
->masked_atomic_cap
= IB_ATOMIC_HCA
;
139 ib_attr
->max_ee_rd_atom
= 0;
140 ib_attr
->max_res_rd_atom
= 0;
141 ib_attr
->max_ee_init_rd_atom
= 0;
143 ib_attr
->max_rdd
= 0;
144 ib_attr
->max_mw
= dev_attr
->max_mw
;
145 ib_attr
->max_raw_ipv6_qp
= 0;
146 ib_attr
->max_raw_ethy_qp
= dev_attr
->max_raw_ethy_qp
;
147 ib_attr
->max_mcast_grp
= 0;
148 ib_attr
->max_mcast_qp_attach
= 0;
149 ib_attr
->max_total_mcast_qp_attach
= 0;
150 ib_attr
->max_ah
= dev_attr
->max_ah
;
152 ib_attr
->max_fmr
= dev_attr
->max_fmr
;
153 ib_attr
->max_map_per_fmr
= 1; /* ? */
155 ib_attr
->max_srq
= dev_attr
->max_srq
;
156 ib_attr
->max_srq_wr
= dev_attr
->max_srq_wqes
;
157 ib_attr
->max_srq_sge
= dev_attr
->max_srq_sges
;
159 ib_attr
->max_fast_reg_page_list_len
= MAX_PBL_LVL_1_PGS
;
161 ib_attr
->max_pkeys
= 1;
162 ib_attr
->local_ca_ack_delay
= 0;
166 int bnxt_re_modify_device(struct ib_device
*ibdev
,
167 int device_modify_mask
,
168 struct ib_device_modify
*device_modify
)
170 switch (device_modify_mask
) {
171 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID
:
172 /* Modify the GUID requires the modification of the GID table */
173 /* GUID should be made as READ-ONLY */
175 case IB_DEVICE_MODIFY_NODE_DESC
:
176 /* Node Desc should be made as READ-ONLY */
184 static void __to_ib_speed_width(struct net_device
*netdev
, u8
*speed
, u8
*width
)
186 struct ethtool_link_ksettings lksettings
;
189 if (netdev
->ethtool_ops
&& netdev
->ethtool_ops
->get_link_ksettings
) {
190 memset(&lksettings
, 0, sizeof(lksettings
));
192 netdev
->ethtool_ops
->get_link_ksettings(netdev
, &lksettings
);
194 espeed
= lksettings
.base
.speed
;
196 espeed
= SPEED_UNKNOWN
;
200 *speed
= IB_SPEED_SDR
;
201 *width
= IB_WIDTH_1X
;
204 *speed
= IB_SPEED_QDR
;
205 *width
= IB_WIDTH_1X
;
208 *speed
= IB_SPEED_DDR
;
209 *width
= IB_WIDTH_4X
;
212 *speed
= IB_SPEED_EDR
;
213 *width
= IB_WIDTH_1X
;
216 *speed
= IB_SPEED_QDR
;
217 *width
= IB_WIDTH_4X
;
222 *speed
= IB_SPEED_SDR
;
223 *width
= IB_WIDTH_1X
;
229 int bnxt_re_query_port(struct ib_device
*ibdev
, u8 port_num
,
230 struct ib_port_attr
*port_attr
)
232 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
233 struct bnxt_qplib_dev_attr
*dev_attr
= &rdev
->dev_attr
;
235 memset(port_attr
, 0, sizeof(*port_attr
));
237 if (netif_running(rdev
->netdev
) && netif_carrier_ok(rdev
->netdev
)) {
238 port_attr
->state
= IB_PORT_ACTIVE
;
239 port_attr
->phys_state
= 5;
241 port_attr
->state
= IB_PORT_DOWN
;
242 port_attr
->phys_state
= 3;
244 port_attr
->max_mtu
= IB_MTU_4096
;
245 port_attr
->active_mtu
= iboe_get_mtu(rdev
->netdev
->mtu
);
246 port_attr
->gid_tbl_len
= dev_attr
->max_sgid
;
247 port_attr
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
248 IB_PORT_DEVICE_MGMT_SUP
|
249 IB_PORT_VENDOR_CLASS_SUP
|
250 IB_PORT_IP_BASED_GIDS
;
252 /* Max MSG size set to 2G for now */
253 port_attr
->max_msg_sz
= 0x80000000;
254 port_attr
->bad_pkey_cntr
= 0;
255 port_attr
->qkey_viol_cntr
= 0;
256 port_attr
->pkey_tbl_len
= dev_attr
->max_pkey
;
258 port_attr
->sm_lid
= 0;
260 port_attr
->max_vl_num
= 4;
261 port_attr
->sm_sl
= 0;
262 port_attr
->subnet_timeout
= 0;
263 port_attr
->init_type_reply
= 0;
264 /* call the underlying netdev's ethtool hooks to query speed settings
265 * for which we acquire rtnl_lock _only_ if it's registered with
266 * IB stack to avoid race in the NETDEV_UNREG path
268 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED
, &rdev
->flags
))
269 __to_ib_speed_width(rdev
->netdev
, &port_attr
->active_speed
,
270 &port_attr
->active_width
);
274 int bnxt_re_modify_port(struct ib_device
*ibdev
, u8 port_num
,
275 int port_modify_mask
,
276 struct ib_port_modify
*port_modify
)
278 switch (port_modify_mask
) {
279 case IB_PORT_SHUTDOWN
:
281 case IB_PORT_INIT_TYPE
:
283 case IB_PORT_RESET_QKEY_CNTR
:
291 int bnxt_re_get_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
292 struct ib_port_immutable
*immutable
)
294 struct ib_port_attr port_attr
;
296 if (bnxt_re_query_port(ibdev
, port_num
, &port_attr
))
299 immutable
->pkey_tbl_len
= port_attr
.pkey_tbl_len
;
300 immutable
->gid_tbl_len
= port_attr
.gid_tbl_len
;
301 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE
;
302 immutable
->core_cap_flags
|= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
;
303 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
307 int bnxt_re_query_pkey(struct ib_device
*ibdev
, u8 port_num
,
308 u16 index
, u16
*pkey
)
310 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
312 /* Ignore port_num */
314 memset(pkey
, 0, sizeof(*pkey
));
315 return bnxt_qplib_get_pkey(&rdev
->qplib_res
,
316 &rdev
->qplib_res
.pkey_tbl
, index
, pkey
);
319 int bnxt_re_query_gid(struct ib_device
*ibdev
, u8 port_num
,
320 int index
, union ib_gid
*gid
)
322 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
325 /* Ignore port_num */
326 memset(gid
, 0, sizeof(*gid
));
327 rc
= bnxt_qplib_get_sgid(&rdev
->qplib_res
,
328 &rdev
->qplib_res
.sgid_tbl
, index
,
329 (struct bnxt_qplib_gid
*)gid
);
333 int bnxt_re_del_gid(struct ib_device
*ibdev
, u8 port_num
,
334 unsigned int index
, void **context
)
337 struct bnxt_re_gid_ctx
*ctx
, **ctx_tbl
;
338 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
339 struct bnxt_qplib_sgid_tbl
*sgid_tbl
= &rdev
->qplib_res
.sgid_tbl
;
341 /* Delete the entry from the hardware */
346 if (sgid_tbl
&& sgid_tbl
->active
) {
347 if (ctx
->idx
>= sgid_tbl
->max
)
351 rc
= bnxt_qplib_del_sgid
353 &sgid_tbl
->tbl
[ctx
->idx
], true);
355 dev_err(rdev_to_dev(rdev
),
356 "Failed to remove GID: %#x", rc
);
357 ctx_tbl
= sgid_tbl
->ctx
;
358 ctx_tbl
[ctx
->idx
] = NULL
;
367 int bnxt_re_add_gid(struct ib_device
*ibdev
, u8 port_num
,
368 unsigned int index
, const union ib_gid
*gid
,
369 const struct ib_gid_attr
*attr
, void **context
)
373 u16 vlan_id
= 0xFFFF;
374 struct bnxt_re_gid_ctx
*ctx
, **ctx_tbl
;
375 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
376 struct bnxt_qplib_sgid_tbl
*sgid_tbl
= &rdev
->qplib_res
.sgid_tbl
;
378 if ((attr
->ndev
) && is_vlan_dev(attr
->ndev
))
379 vlan_id
= vlan_dev_vlan_id(attr
->ndev
);
381 rc
= bnxt_qplib_add_sgid(sgid_tbl
, (struct bnxt_qplib_gid
*)gid
,
382 rdev
->qplib_res
.netdev
->dev_addr
,
383 vlan_id
, true, &tbl_idx
);
384 if (rc
== -EALREADY
) {
385 ctx_tbl
= sgid_tbl
->ctx
;
386 ctx_tbl
[tbl_idx
]->refcnt
++;
387 *context
= ctx_tbl
[tbl_idx
];
392 dev_err(rdev_to_dev(rdev
), "Failed to add GID: %#x", rc
);
396 ctx
= kmalloc(sizeof(*ctx
), GFP_KERNEL
);
399 ctx_tbl
= sgid_tbl
->ctx
;
402 ctx_tbl
[tbl_idx
] = ctx
;
407 enum rdma_link_layer
bnxt_re_get_link_layer(struct ib_device
*ibdev
,
410 return IB_LINK_LAYER_ETHERNET
;
413 /* Protection Domains */
414 int bnxt_re_dealloc_pd(struct ib_pd
*ib_pd
)
416 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
417 struct bnxt_re_dev
*rdev
= pd
->rdev
;
420 if (ib_pd
->uobject
&& pd
->dpi
.dbr
) {
421 struct ib_ucontext
*ib_uctx
= ib_pd
->uobject
->context
;
422 struct bnxt_re_ucontext
*ucntx
;
424 /* Free DPI only if this is the first PD allocated by the
425 * application and mark the context dpi as NULL
427 ucntx
= container_of(ib_uctx
, struct bnxt_re_ucontext
, ib_uctx
);
429 rc
= bnxt_qplib_dealloc_dpi(&rdev
->qplib_res
,
430 &rdev
->qplib_res
.dpi_tbl
,
433 dev_err(rdev_to_dev(rdev
), "Failed to deallocate HW DPI");
434 /* Don't fail, continue*/
438 rc
= bnxt_qplib_dealloc_pd(&rdev
->qplib_res
,
439 &rdev
->qplib_res
.pd_tbl
,
442 dev_err(rdev_to_dev(rdev
), "Failed to deallocate HW PD");
450 struct ib_pd
*bnxt_re_alloc_pd(struct ib_device
*ibdev
,
451 struct ib_ucontext
*ucontext
,
452 struct ib_udata
*udata
)
454 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
455 struct bnxt_re_ucontext
*ucntx
= container_of(ucontext
,
456 struct bnxt_re_ucontext
,
458 struct bnxt_re_pd
*pd
;
461 pd
= kzalloc(sizeof(*pd
), GFP_KERNEL
);
463 return ERR_PTR(-ENOMEM
);
466 if (bnxt_qplib_alloc_pd(&rdev
->qplib_res
.pd_tbl
, &pd
->qplib_pd
)) {
467 dev_err(rdev_to_dev(rdev
), "Failed to allocate HW PD");
473 struct bnxt_re_pd_resp resp
;
476 /* Allocate DPI in alloc_pd to avoid failing of
477 * ibv_devinfo and family of application when DPIs
480 if (bnxt_qplib_alloc_dpi(&rdev
->qplib_res
.dpi_tbl
,
485 ucntx
->dpi
= &pd
->dpi
;
488 resp
.pdid
= pd
->qplib_pd
.id
;
489 /* Still allow mapping this DBR to the new user PD. */
490 resp
.dpi
= ucntx
->dpi
->dpi
;
491 resp
.dbr
= (u64
)ucntx
->dpi
->umdbr
;
493 rc
= ib_copy_to_udata(udata
, &resp
, sizeof(resp
));
495 dev_err(rdev_to_dev(rdev
),
496 "Failed to copy user response\n");
503 (void)bnxt_qplib_dealloc_pd(&rdev
->qplib_res
, &rdev
->qplib_res
.pd_tbl
,
510 /* Address Handles */
511 int bnxt_re_destroy_ah(struct ib_ah
*ib_ah
)
513 struct bnxt_re_ah
*ah
= container_of(ib_ah
, struct bnxt_re_ah
, ib_ah
);
514 struct bnxt_re_dev
*rdev
= ah
->rdev
;
517 rc
= bnxt_qplib_destroy_ah(&rdev
->qplib_res
, &ah
->qplib_ah
);
519 dev_err(rdev_to_dev(rdev
), "Failed to destroy HW AH");
526 struct ib_ah
*bnxt_re_create_ah(struct ib_pd
*ib_pd
,
527 struct rdma_ah_attr
*ah_attr
,
528 struct ib_udata
*udata
)
530 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
531 struct bnxt_re_dev
*rdev
= pd
->rdev
;
532 struct bnxt_re_ah
*ah
;
533 const struct ib_global_route
*grh
= rdma_ah_read_grh(ah_attr
);
538 struct ib_gid_attr sgid_attr
;
540 if (!(rdma_ah_get_ah_flags(ah_attr
) & IB_AH_GRH
)) {
541 dev_err(rdev_to_dev(rdev
), "Failed to alloc AH: GRH not set");
542 return ERR_PTR(-EINVAL
);
544 ah
= kzalloc(sizeof(*ah
), GFP_ATOMIC
);
546 return ERR_PTR(-ENOMEM
);
549 ah
->qplib_ah
.pd
= &pd
->qplib_pd
;
551 /* Supply the configuration for the HW */
552 memcpy(ah
->qplib_ah
.dgid
.data
, grh
->dgid
.raw
,
553 sizeof(union ib_gid
));
555 * If RoCE V2 is enabled, stack will have two entries for
556 * each GID entry. Avoiding this duplicte entry in HW. Dividing
557 * the GID index by 2 for RoCE V2
559 ah
->qplib_ah
.sgid_index
= grh
->sgid_index
/ 2;
560 ah
->qplib_ah
.host_sgid_index
= grh
->sgid_index
;
561 ah
->qplib_ah
.traffic_class
= grh
->traffic_class
;
562 ah
->qplib_ah
.flow_label
= grh
->flow_label
;
563 ah
->qplib_ah
.hop_limit
= grh
->hop_limit
;
564 ah
->qplib_ah
.sl
= rdma_ah_get_sl(ah_attr
);
565 if (ib_pd
->uobject
&&
566 !rdma_is_multicast_addr((struct in6_addr
*)
568 !rdma_link_local_addr((struct in6_addr
*)
572 rc
= ib_get_cached_gid(&rdev
->ibdev
, 1,
573 grh
->sgid_index
, &sgid
,
576 dev_err(rdev_to_dev(rdev
),
577 "Failed to query gid at index %d",
581 if (sgid_attr
.ndev
) {
582 if (is_vlan_dev(sgid_attr
.ndev
))
583 vlan_tag
= vlan_dev_vlan_id(sgid_attr
.ndev
);
584 dev_put(sgid_attr
.ndev
);
586 /* Get network header type for this GID */
587 nw_type
= ib_gid_to_network_type(sgid_attr
.gid_type
, &sgid
);
589 case RDMA_NETWORK_IPV4
:
590 ah
->qplib_ah
.nw_type
= CMDQ_CREATE_AH_TYPE_V2IPV4
;
592 case RDMA_NETWORK_IPV6
:
593 ah
->qplib_ah
.nw_type
= CMDQ_CREATE_AH_TYPE_V2IPV6
;
596 ah
->qplib_ah
.nw_type
= CMDQ_CREATE_AH_TYPE_V1
;
599 rc
= rdma_addr_find_l2_eth_by_grh(&sgid
, &grh
->dgid
,
600 ah_attr
->roce
.dmac
, &vlan_tag
,
601 &sgid_attr
.ndev
->ifindex
,
604 dev_err(rdev_to_dev(rdev
), "Failed to get dmac\n");
609 memcpy(ah
->qplib_ah
.dmac
, ah_attr
->roce
.dmac
, ETH_ALEN
);
610 rc
= bnxt_qplib_create_ah(&rdev
->qplib_res
, &ah
->qplib_ah
);
612 dev_err(rdev_to_dev(rdev
), "Failed to allocate HW AH");
616 /* Write AVID to shared page. */
617 if (ib_pd
->uobject
) {
618 struct ib_ucontext
*ib_uctx
= ib_pd
->uobject
->context
;
619 struct bnxt_re_ucontext
*uctx
;
623 uctx
= container_of(ib_uctx
, struct bnxt_re_ucontext
, ib_uctx
);
624 spin_lock_irqsave(&uctx
->sh_lock
, flag
);
625 wrptr
= (u32
*)(uctx
->shpg
+ BNXT_RE_AVID_OFFT
);
626 *wrptr
= ah
->qplib_ah
.id
;
627 wmb(); /* make sure cache is updated. */
628 spin_unlock_irqrestore(&uctx
->sh_lock
, flag
);
638 int bnxt_re_modify_ah(struct ib_ah
*ib_ah
, struct rdma_ah_attr
*ah_attr
)
643 int bnxt_re_query_ah(struct ib_ah
*ib_ah
, struct rdma_ah_attr
*ah_attr
)
645 struct bnxt_re_ah
*ah
= container_of(ib_ah
, struct bnxt_re_ah
, ib_ah
);
647 ah_attr
->type
= ib_ah
->type
;
648 rdma_ah_set_sl(ah_attr
, ah
->qplib_ah
.sl
);
649 memcpy(ah_attr
->roce
.dmac
, ah
->qplib_ah
.dmac
, ETH_ALEN
);
650 rdma_ah_set_grh(ah_attr
, NULL
, 0,
651 ah
->qplib_ah
.host_sgid_index
,
652 0, ah
->qplib_ah
.traffic_class
);
653 rdma_ah_set_dgid_raw(ah_attr
, ah
->qplib_ah
.dgid
.data
);
654 rdma_ah_set_port_num(ah_attr
, 1);
655 rdma_ah_set_static_rate(ah_attr
, 0);
660 int bnxt_re_destroy_qp(struct ib_qp
*ib_qp
)
662 struct bnxt_re_qp
*qp
= container_of(ib_qp
, struct bnxt_re_qp
, ib_qp
);
663 struct bnxt_re_dev
*rdev
= qp
->rdev
;
666 rc
= bnxt_qplib_destroy_qp(&rdev
->qplib_res
, &qp
->qplib_qp
);
668 dev_err(rdev_to_dev(rdev
), "Failed to destroy HW QP");
671 if (ib_qp
->qp_type
== IB_QPT_GSI
&& rdev
->qp1_sqp
) {
672 rc
= bnxt_qplib_destroy_ah(&rdev
->qplib_res
,
673 &rdev
->sqp_ah
->qplib_ah
);
675 dev_err(rdev_to_dev(rdev
),
676 "Failed to destroy HW AH for shadow QP");
680 rc
= bnxt_qplib_destroy_qp(&rdev
->qplib_res
,
681 &rdev
->qp1_sqp
->qplib_qp
);
683 dev_err(rdev_to_dev(rdev
),
684 "Failed to destroy Shadow QP");
687 mutex_lock(&rdev
->qp_lock
);
688 list_del(&rdev
->qp1_sqp
->list
);
689 atomic_dec(&rdev
->qp_count
);
690 mutex_unlock(&rdev
->qp_lock
);
693 kfree(rdev
->qp1_sqp
);
696 if (!IS_ERR_OR_NULL(qp
->rumem
))
697 ib_umem_release(qp
->rumem
);
698 if (!IS_ERR_OR_NULL(qp
->sumem
))
699 ib_umem_release(qp
->sumem
);
701 mutex_lock(&rdev
->qp_lock
);
703 atomic_dec(&rdev
->qp_count
);
704 mutex_unlock(&rdev
->qp_lock
);
709 static u8
__from_ib_qp_type(enum ib_qp_type type
)
713 return CMDQ_CREATE_QP1_TYPE_GSI
;
715 return CMDQ_CREATE_QP_TYPE_RC
;
717 return CMDQ_CREATE_QP_TYPE_UD
;
723 static int bnxt_re_init_user_qp(struct bnxt_re_dev
*rdev
, struct bnxt_re_pd
*pd
,
724 struct bnxt_re_qp
*qp
, struct ib_udata
*udata
)
726 struct bnxt_re_qp_req ureq
;
727 struct bnxt_qplib_qp
*qplib_qp
= &qp
->qplib_qp
;
728 struct ib_umem
*umem
;
730 struct ib_ucontext
*context
= pd
->ib_pd
.uobject
->context
;
731 struct bnxt_re_ucontext
*cntx
= container_of(context
,
732 struct bnxt_re_ucontext
,
734 if (ib_copy_from_udata(&ureq
, udata
, sizeof(ureq
)))
737 bytes
= (qplib_qp
->sq
.max_wqe
* BNXT_QPLIB_MAX_SQE_ENTRY_SIZE
);
738 /* Consider mapping PSN search memory only for RC QPs. */
739 if (qplib_qp
->type
== CMDQ_CREATE_QP_TYPE_RC
)
740 bytes
+= (qplib_qp
->sq
.max_wqe
* sizeof(struct sq_psn_search
));
741 bytes
= PAGE_ALIGN(bytes
);
742 umem
= ib_umem_get(context
, ureq
.qpsva
, bytes
,
743 IB_ACCESS_LOCAL_WRITE
, 1);
745 return PTR_ERR(umem
);
748 qplib_qp
->sq
.sglist
= umem
->sg_head
.sgl
;
749 qplib_qp
->sq
.nmap
= umem
->nmap
;
750 qplib_qp
->qp_handle
= ureq
.qp_handle
;
752 if (!qp
->qplib_qp
.srq
) {
753 bytes
= (qplib_qp
->rq
.max_wqe
* BNXT_QPLIB_MAX_RQE_ENTRY_SIZE
);
754 bytes
= PAGE_ALIGN(bytes
);
755 umem
= ib_umem_get(context
, ureq
.qprva
, bytes
,
756 IB_ACCESS_LOCAL_WRITE
, 1);
760 qplib_qp
->rq
.sglist
= umem
->sg_head
.sgl
;
761 qplib_qp
->rq
.nmap
= umem
->nmap
;
764 qplib_qp
->dpi
= cntx
->dpi
;
767 ib_umem_release(qp
->sumem
);
769 qplib_qp
->sq
.sglist
= NULL
;
770 qplib_qp
->sq
.nmap
= 0;
772 return PTR_ERR(umem
);
775 static struct bnxt_re_ah
*bnxt_re_create_shadow_qp_ah
776 (struct bnxt_re_pd
*pd
,
777 struct bnxt_qplib_res
*qp1_res
,
778 struct bnxt_qplib_qp
*qp1_qp
)
780 struct bnxt_re_dev
*rdev
= pd
->rdev
;
781 struct bnxt_re_ah
*ah
;
785 ah
= kzalloc(sizeof(*ah
), GFP_KERNEL
);
789 memset(ah
, 0, sizeof(*ah
));
791 ah
->qplib_ah
.pd
= &pd
->qplib_pd
;
793 rc
= bnxt_re_query_gid(&rdev
->ibdev
, 1, 0, &sgid
);
797 /* supply the dgid data same as sgid */
798 memcpy(ah
->qplib_ah
.dgid
.data
, &sgid
.raw
,
799 sizeof(union ib_gid
));
800 ah
->qplib_ah
.sgid_index
= 0;
802 ah
->qplib_ah
.traffic_class
= 0;
803 ah
->qplib_ah
.flow_label
= 0;
804 ah
->qplib_ah
.hop_limit
= 1;
806 /* Have DMAC same as SMAC */
807 ether_addr_copy(ah
->qplib_ah
.dmac
, rdev
->netdev
->dev_addr
);
809 rc
= bnxt_qplib_create_ah(&rdev
->qplib_res
, &ah
->qplib_ah
);
811 dev_err(rdev_to_dev(rdev
),
812 "Failed to allocate HW AH for Shadow QP");
823 static struct bnxt_re_qp
*bnxt_re_create_shadow_qp
824 (struct bnxt_re_pd
*pd
,
825 struct bnxt_qplib_res
*qp1_res
,
826 struct bnxt_qplib_qp
*qp1_qp
)
828 struct bnxt_re_dev
*rdev
= pd
->rdev
;
829 struct bnxt_re_qp
*qp
;
832 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
836 memset(qp
, 0, sizeof(*qp
));
839 /* Initialize the shadow QP structure from the QP1 values */
840 ether_addr_copy(qp
->qplib_qp
.smac
, rdev
->netdev
->dev_addr
);
842 qp
->qplib_qp
.pd
= &pd
->qplib_pd
;
843 qp
->qplib_qp
.qp_handle
= (u64
)(unsigned long)(&qp
->qplib_qp
);
844 qp
->qplib_qp
.type
= IB_QPT_UD
;
846 qp
->qplib_qp
.max_inline_data
= 0;
847 qp
->qplib_qp
.sig_type
= true;
849 /* Shadow QP SQ depth should be same as QP1 RQ depth */
850 qp
->qplib_qp
.sq
.max_wqe
= qp1_qp
->rq
.max_wqe
;
851 qp
->qplib_qp
.sq
.max_sge
= 2;
853 qp
->qplib_qp
.scq
= qp1_qp
->scq
;
854 qp
->qplib_qp
.rcq
= qp1_qp
->rcq
;
856 qp
->qplib_qp
.rq
.max_wqe
= qp1_qp
->rq
.max_wqe
;
857 qp
->qplib_qp
.rq
.max_sge
= qp1_qp
->rq
.max_sge
;
859 qp
->qplib_qp
.mtu
= qp1_qp
->mtu
;
861 qp
->qplib_qp
.sq_hdr_buf_size
= 0;
862 qp
->qplib_qp
.rq_hdr_buf_size
= BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6
;
863 qp
->qplib_qp
.dpi
= &rdev
->dpi_privileged
;
865 rc
= bnxt_qplib_create_qp(qp1_res
, &qp
->qplib_qp
);
869 rdev
->sqp_id
= qp
->qplib_qp
.id
;
871 spin_lock_init(&qp
->sq_lock
);
872 INIT_LIST_HEAD(&qp
->list
);
873 mutex_lock(&rdev
->qp_lock
);
874 list_add_tail(&qp
->list
, &rdev
->qp_list
);
875 atomic_inc(&rdev
->qp_count
);
876 mutex_unlock(&rdev
->qp_lock
);
883 struct ib_qp
*bnxt_re_create_qp(struct ib_pd
*ib_pd
,
884 struct ib_qp_init_attr
*qp_init_attr
,
885 struct ib_udata
*udata
)
887 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
888 struct bnxt_re_dev
*rdev
= pd
->rdev
;
889 struct bnxt_qplib_dev_attr
*dev_attr
= &rdev
->dev_attr
;
890 struct bnxt_re_qp
*qp
;
891 struct bnxt_re_cq
*cq
;
894 if ((qp_init_attr
->cap
.max_send_wr
> dev_attr
->max_qp_wqes
) ||
895 (qp_init_attr
->cap
.max_recv_wr
> dev_attr
->max_qp_wqes
) ||
896 (qp_init_attr
->cap
.max_send_sge
> dev_attr
->max_qp_sges
) ||
897 (qp_init_attr
->cap
.max_recv_sge
> dev_attr
->max_qp_sges
) ||
898 (qp_init_attr
->cap
.max_inline_data
> dev_attr
->max_inline_data
))
899 return ERR_PTR(-EINVAL
);
901 qp
= kzalloc(sizeof(*qp
), GFP_KERNEL
);
903 return ERR_PTR(-ENOMEM
);
906 ether_addr_copy(qp
->qplib_qp
.smac
, rdev
->netdev
->dev_addr
);
907 qp
->qplib_qp
.pd
= &pd
->qplib_pd
;
908 qp
->qplib_qp
.qp_handle
= (u64
)(unsigned long)(&qp
->qplib_qp
);
909 qp
->qplib_qp
.type
= __from_ib_qp_type(qp_init_attr
->qp_type
);
910 if (qp
->qplib_qp
.type
== IB_QPT_MAX
) {
911 dev_err(rdev_to_dev(rdev
), "QP type 0x%x not supported",
916 qp
->qplib_qp
.max_inline_data
= qp_init_attr
->cap
.max_inline_data
;
917 qp
->qplib_qp
.sig_type
= ((qp_init_attr
->sq_sig_type
==
918 IB_SIGNAL_ALL_WR
) ? true : false);
920 entries
= roundup_pow_of_two(qp_init_attr
->cap
.max_send_wr
+ 1);
921 qp
->qplib_qp
.sq
.max_wqe
= min_t(u32
, entries
,
922 dev_attr
->max_qp_wqes
+ 1);
924 qp
->qplib_qp
.sq
.max_sge
= qp_init_attr
->cap
.max_send_sge
;
925 if (qp
->qplib_qp
.sq
.max_sge
> dev_attr
->max_qp_sges
)
926 qp
->qplib_qp
.sq
.max_sge
= dev_attr
->max_qp_sges
;
928 if (qp_init_attr
->send_cq
) {
929 cq
= container_of(qp_init_attr
->send_cq
, struct bnxt_re_cq
,
932 dev_err(rdev_to_dev(rdev
), "Send CQ not found");
936 qp
->qplib_qp
.scq
= &cq
->qplib_cq
;
939 if (qp_init_attr
->recv_cq
) {
940 cq
= container_of(qp_init_attr
->recv_cq
, struct bnxt_re_cq
,
943 dev_err(rdev_to_dev(rdev
), "Receive CQ not found");
947 qp
->qplib_qp
.rcq
= &cq
->qplib_cq
;
950 if (qp_init_attr
->srq
) {
951 dev_err(rdev_to_dev(rdev
), "SRQ not supported");
955 /* Allocate 1 more than what's provided so posting max doesn't
958 entries
= roundup_pow_of_two(qp_init_attr
->cap
.max_recv_wr
+ 1);
959 qp
->qplib_qp
.rq
.max_wqe
= min_t(u32
, entries
,
960 dev_attr
->max_qp_wqes
+ 1);
962 qp
->qplib_qp
.rq
.max_sge
= qp_init_attr
->cap
.max_recv_sge
;
963 if (qp
->qplib_qp
.rq
.max_sge
> dev_attr
->max_qp_sges
)
964 qp
->qplib_qp
.rq
.max_sge
= dev_attr
->max_qp_sges
;
967 qp
->qplib_qp
.mtu
= ib_mtu_enum_to_int(iboe_get_mtu(rdev
->netdev
->mtu
));
969 if (qp_init_attr
->qp_type
== IB_QPT_GSI
) {
970 qp
->qplib_qp
.rq
.max_sge
= dev_attr
->max_qp_sges
;
971 if (qp
->qplib_qp
.rq
.max_sge
> dev_attr
->max_qp_sges
)
972 qp
->qplib_qp
.rq
.max_sge
= dev_attr
->max_qp_sges
;
973 qp
->qplib_qp
.sq
.max_sge
++;
974 if (qp
->qplib_qp
.sq
.max_sge
> dev_attr
->max_qp_sges
)
975 qp
->qplib_qp
.sq
.max_sge
= dev_attr
->max_qp_sges
;
977 qp
->qplib_qp
.rq_hdr_buf_size
=
978 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2
;
980 qp
->qplib_qp
.sq_hdr_buf_size
=
981 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2
;
982 qp
->qplib_qp
.dpi
= &rdev
->dpi_privileged
;
983 rc
= bnxt_qplib_create_qp1(&rdev
->qplib_res
, &qp
->qplib_qp
);
985 dev_err(rdev_to_dev(rdev
), "Failed to create HW QP1");
988 /* Create a shadow QP to handle the QP1 traffic */
989 rdev
->qp1_sqp
= bnxt_re_create_shadow_qp(pd
, &rdev
->qplib_res
,
991 if (!rdev
->qp1_sqp
) {
993 dev_err(rdev_to_dev(rdev
),
994 "Failed to create Shadow QP for QP1");
997 rdev
->sqp_ah
= bnxt_re_create_shadow_qp_ah(pd
, &rdev
->qplib_res
,
1000 bnxt_qplib_destroy_qp(&rdev
->qplib_res
,
1001 &rdev
->qp1_sqp
->qplib_qp
);
1003 dev_err(rdev_to_dev(rdev
),
1004 "Failed to create AH entry for ShadowQP");
1009 qp
->qplib_qp
.max_rd_atomic
= dev_attr
->max_qp_rd_atom
;
1010 qp
->qplib_qp
.max_dest_rd_atomic
= dev_attr
->max_qp_init_rd_atom
;
1012 rc
= bnxt_re_init_user_qp(rdev
, pd
, qp
, udata
);
1016 qp
->qplib_qp
.dpi
= &rdev
->dpi_privileged
;
1019 rc
= bnxt_qplib_create_qp(&rdev
->qplib_res
, &qp
->qplib_qp
);
1021 dev_err(rdev_to_dev(rdev
), "Failed to create HW QP");
1026 qp
->ib_qp
.qp_num
= qp
->qplib_qp
.id
;
1027 spin_lock_init(&qp
->sq_lock
);
1030 struct bnxt_re_qp_resp resp
;
1032 resp
.qpid
= qp
->ib_qp
.qp_num
;
1034 rc
= ib_copy_to_udata(udata
, &resp
, sizeof(resp
));
1036 dev_err(rdev_to_dev(rdev
), "Failed to copy QP udata");
1040 INIT_LIST_HEAD(&qp
->list
);
1041 mutex_lock(&rdev
->qp_lock
);
1042 list_add_tail(&qp
->list
, &rdev
->qp_list
);
1043 atomic_inc(&rdev
->qp_count
);
1044 mutex_unlock(&rdev
->qp_lock
);
1048 bnxt_qplib_destroy_qp(&rdev
->qplib_res
, &qp
->qplib_qp
);
1054 static u8
__from_ib_qp_state(enum ib_qp_state state
)
1058 return CMDQ_MODIFY_QP_NEW_STATE_RESET
;
1060 return CMDQ_MODIFY_QP_NEW_STATE_INIT
;
1062 return CMDQ_MODIFY_QP_NEW_STATE_RTR
;
1064 return CMDQ_MODIFY_QP_NEW_STATE_RTS
;
1066 return CMDQ_MODIFY_QP_NEW_STATE_SQD
;
1068 return CMDQ_MODIFY_QP_NEW_STATE_SQE
;
1071 return CMDQ_MODIFY_QP_NEW_STATE_ERR
;
1075 static enum ib_qp_state
__to_ib_qp_state(u8 state
)
1078 case CMDQ_MODIFY_QP_NEW_STATE_RESET
:
1079 return IB_QPS_RESET
;
1080 case CMDQ_MODIFY_QP_NEW_STATE_INIT
:
1082 case CMDQ_MODIFY_QP_NEW_STATE_RTR
:
1084 case CMDQ_MODIFY_QP_NEW_STATE_RTS
:
1086 case CMDQ_MODIFY_QP_NEW_STATE_SQD
:
1088 case CMDQ_MODIFY_QP_NEW_STATE_SQE
:
1090 case CMDQ_MODIFY_QP_NEW_STATE_ERR
:
1096 static u32
__from_ib_mtu(enum ib_mtu mtu
)
1100 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256
;
1102 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512
;
1104 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024
;
1106 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048
;
1108 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096
;
1110 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048
;
1114 static enum ib_mtu
__to_ib_mtu(u32 mtu
)
1116 switch (mtu
& CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK
) {
1117 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256
:
1119 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512
:
1121 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024
:
1123 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048
:
1125 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096
:
1132 static int __from_ib_access_flags(int iflags
)
1136 if (iflags
& IB_ACCESS_LOCAL_WRITE
)
1137 qflags
|= BNXT_QPLIB_ACCESS_LOCAL_WRITE
;
1138 if (iflags
& IB_ACCESS_REMOTE_READ
)
1139 qflags
|= BNXT_QPLIB_ACCESS_REMOTE_READ
;
1140 if (iflags
& IB_ACCESS_REMOTE_WRITE
)
1141 qflags
|= BNXT_QPLIB_ACCESS_REMOTE_WRITE
;
1142 if (iflags
& IB_ACCESS_REMOTE_ATOMIC
)
1143 qflags
|= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC
;
1144 if (iflags
& IB_ACCESS_MW_BIND
)
1145 qflags
|= BNXT_QPLIB_ACCESS_MW_BIND
;
1146 if (iflags
& IB_ZERO_BASED
)
1147 qflags
|= BNXT_QPLIB_ACCESS_ZERO_BASED
;
1148 if (iflags
& IB_ACCESS_ON_DEMAND
)
1149 qflags
|= BNXT_QPLIB_ACCESS_ON_DEMAND
;
1153 static enum ib_access_flags
__to_ib_access_flags(int qflags
)
1155 enum ib_access_flags iflags
= 0;
1157 if (qflags
& BNXT_QPLIB_ACCESS_LOCAL_WRITE
)
1158 iflags
|= IB_ACCESS_LOCAL_WRITE
;
1159 if (qflags
& BNXT_QPLIB_ACCESS_REMOTE_WRITE
)
1160 iflags
|= IB_ACCESS_REMOTE_WRITE
;
1161 if (qflags
& BNXT_QPLIB_ACCESS_REMOTE_READ
)
1162 iflags
|= IB_ACCESS_REMOTE_READ
;
1163 if (qflags
& BNXT_QPLIB_ACCESS_REMOTE_ATOMIC
)
1164 iflags
|= IB_ACCESS_REMOTE_ATOMIC
;
1165 if (qflags
& BNXT_QPLIB_ACCESS_MW_BIND
)
1166 iflags
|= IB_ACCESS_MW_BIND
;
1167 if (qflags
& BNXT_QPLIB_ACCESS_ZERO_BASED
)
1168 iflags
|= IB_ZERO_BASED
;
1169 if (qflags
& BNXT_QPLIB_ACCESS_ON_DEMAND
)
1170 iflags
|= IB_ACCESS_ON_DEMAND
;
1174 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev
*rdev
,
1175 struct bnxt_re_qp
*qp1_qp
,
1178 struct bnxt_re_qp
*qp
= rdev
->qp1_sqp
;
1181 if (qp_attr_mask
& IB_QP_STATE
) {
1182 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_STATE
;
1183 qp
->qplib_qp
.state
= qp1_qp
->qplib_qp
.state
;
1185 if (qp_attr_mask
& IB_QP_PKEY_INDEX
) {
1186 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
;
1187 qp
->qplib_qp
.pkey_index
= qp1_qp
->qplib_qp
.pkey_index
;
1190 if (qp_attr_mask
& IB_QP_QKEY
) {
1191 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
;
1192 /* Using a Random QKEY */
1193 qp
->qplib_qp
.qkey
= 0x81818181;
1195 if (qp_attr_mask
& IB_QP_SQ_PSN
) {
1196 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN
;
1197 qp
->qplib_qp
.sq
.psn
= qp1_qp
->qplib_qp
.sq
.psn
;
1200 rc
= bnxt_qplib_modify_qp(&rdev
->qplib_res
, &qp
->qplib_qp
);
1202 dev_err(rdev_to_dev(rdev
),
1203 "Failed to modify Shadow QP for QP1");
1207 int bnxt_re_modify_qp(struct ib_qp
*ib_qp
, struct ib_qp_attr
*qp_attr
,
1208 int qp_attr_mask
, struct ib_udata
*udata
)
1210 struct bnxt_re_qp
*qp
= container_of(ib_qp
, struct bnxt_re_qp
, ib_qp
);
1211 struct bnxt_re_dev
*rdev
= qp
->rdev
;
1212 struct bnxt_qplib_dev_attr
*dev_attr
= &rdev
->dev_attr
;
1213 enum ib_qp_state curr_qp_state
, new_qp_state
;
1217 struct ib_gid_attr sgid_attr
;
1220 qp
->qplib_qp
.modify_flags
= 0;
1221 if (qp_attr_mask
& IB_QP_STATE
) {
1222 curr_qp_state
= __to_ib_qp_state(qp
->qplib_qp
.cur_qp_state
);
1223 new_qp_state
= qp_attr
->qp_state
;
1224 if (!ib_modify_qp_is_ok(curr_qp_state
, new_qp_state
,
1225 ib_qp
->qp_type
, qp_attr_mask
,
1226 IB_LINK_LAYER_ETHERNET
)) {
1227 dev_err(rdev_to_dev(rdev
),
1228 "Invalid attribute mask: %#x specified ",
1230 dev_err(rdev_to_dev(rdev
),
1231 "for qpn: %#x type: %#x",
1232 ib_qp
->qp_num
, ib_qp
->qp_type
);
1233 dev_err(rdev_to_dev(rdev
),
1234 "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1235 curr_qp_state
, new_qp_state
);
1238 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_STATE
;
1239 qp
->qplib_qp
.state
= __from_ib_qp_state(qp_attr
->qp_state
);
1241 if (qp_attr_mask
& IB_QP_EN_SQD_ASYNC_NOTIFY
) {
1242 qp
->qplib_qp
.modify_flags
|=
1243 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY
;
1244 qp
->qplib_qp
.en_sqd_async_notify
= true;
1246 if (qp_attr_mask
& IB_QP_ACCESS_FLAGS
) {
1247 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS
;
1248 qp
->qplib_qp
.access
=
1249 __from_ib_access_flags(qp_attr
->qp_access_flags
);
1250 /* LOCAL_WRITE access must be set to allow RC receive */
1251 qp
->qplib_qp
.access
|= BNXT_QPLIB_ACCESS_LOCAL_WRITE
;
1253 if (qp_attr_mask
& IB_QP_PKEY_INDEX
) {
1254 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY
;
1255 qp
->qplib_qp
.pkey_index
= qp_attr
->pkey_index
;
1257 if (qp_attr_mask
& IB_QP_QKEY
) {
1258 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY
;
1259 qp
->qplib_qp
.qkey
= qp_attr
->qkey
;
1261 if (qp_attr_mask
& IB_QP_AV
) {
1262 const struct ib_global_route
*grh
=
1263 rdma_ah_read_grh(&qp_attr
->ah_attr
);
1265 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_DGID
|
1266 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL
|
1267 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX
|
1268 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT
|
1269 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS
|
1270 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC
|
1271 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID
;
1272 memcpy(qp
->qplib_qp
.ah
.dgid
.data
, grh
->dgid
.raw
,
1273 sizeof(qp
->qplib_qp
.ah
.dgid
.data
));
1274 qp
->qplib_qp
.ah
.flow_label
= grh
->flow_label
;
1275 /* If RoCE V2 is enabled, stack will have two entries for
1276 * each GID entry. Avoiding this duplicte entry in HW. Dividing
1277 * the GID index by 2 for RoCE V2
1279 qp
->qplib_qp
.ah
.sgid_index
= grh
->sgid_index
/ 2;
1280 qp
->qplib_qp
.ah
.host_sgid_index
= grh
->sgid_index
;
1281 qp
->qplib_qp
.ah
.hop_limit
= grh
->hop_limit
;
1282 qp
->qplib_qp
.ah
.traffic_class
= grh
->traffic_class
;
1283 qp
->qplib_qp
.ah
.sl
= rdma_ah_get_sl(&qp_attr
->ah_attr
);
1284 ether_addr_copy(qp
->qplib_qp
.ah
.dmac
,
1285 qp_attr
->ah_attr
.roce
.dmac
);
1287 status
= ib_get_cached_gid(&rdev
->ibdev
, 1,
1290 if (!status
&& sgid_attr
.ndev
) {
1291 memcpy(qp
->qplib_qp
.smac
, sgid_attr
.ndev
->dev_addr
,
1293 dev_put(sgid_attr
.ndev
);
1294 nw_type
= ib_gid_to_network_type(sgid_attr
.gid_type
,
1297 case RDMA_NETWORK_IPV4
:
1298 qp
->qplib_qp
.nw_type
=
1299 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4
;
1301 case RDMA_NETWORK_IPV6
:
1302 qp
->qplib_qp
.nw_type
=
1303 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6
;
1306 qp
->qplib_qp
.nw_type
=
1307 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1
;
1313 if (qp_attr_mask
& IB_QP_PATH_MTU
) {
1314 qp
->qplib_qp
.modify_flags
|=
1315 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU
;
1316 qp
->qplib_qp
.path_mtu
= __from_ib_mtu(qp_attr
->path_mtu
);
1317 } else if (qp_attr
->qp_state
== IB_QPS_RTR
) {
1318 qp
->qplib_qp
.modify_flags
|=
1319 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU
;
1320 qp
->qplib_qp
.path_mtu
=
1321 __from_ib_mtu(iboe_get_mtu(rdev
->netdev
->mtu
));
1324 if (qp_attr_mask
& IB_QP_TIMEOUT
) {
1325 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT
;
1326 qp
->qplib_qp
.timeout
= qp_attr
->timeout
;
1328 if (qp_attr_mask
& IB_QP_RETRY_CNT
) {
1329 qp
->qplib_qp
.modify_flags
|=
1330 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT
;
1331 qp
->qplib_qp
.retry_cnt
= qp_attr
->retry_cnt
;
1333 if (qp_attr_mask
& IB_QP_RNR_RETRY
) {
1334 qp
->qplib_qp
.modify_flags
|=
1335 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY
;
1336 qp
->qplib_qp
.rnr_retry
= qp_attr
->rnr_retry
;
1338 if (qp_attr_mask
& IB_QP_MIN_RNR_TIMER
) {
1339 qp
->qplib_qp
.modify_flags
|=
1340 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER
;
1341 qp
->qplib_qp
.min_rnr_timer
= qp_attr
->min_rnr_timer
;
1343 if (qp_attr_mask
& IB_QP_RQ_PSN
) {
1344 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN
;
1345 qp
->qplib_qp
.rq
.psn
= qp_attr
->rq_psn
;
1347 if (qp_attr_mask
& IB_QP_MAX_QP_RD_ATOMIC
) {
1348 qp
->qplib_qp
.modify_flags
|=
1349 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC
;
1350 qp
->qplib_qp
.max_rd_atomic
= qp_attr
->max_rd_atomic
;
1352 if (qp_attr_mask
& IB_QP_SQ_PSN
) {
1353 qp
->qplib_qp
.modify_flags
|= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN
;
1354 qp
->qplib_qp
.sq
.psn
= qp_attr
->sq_psn
;
1356 if (qp_attr_mask
& IB_QP_MAX_DEST_RD_ATOMIC
) {
1357 qp
->qplib_qp
.modify_flags
|=
1358 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC
;
1359 qp
->qplib_qp
.max_dest_rd_atomic
= qp_attr
->max_dest_rd_atomic
;
1361 if (qp_attr_mask
& IB_QP_CAP
) {
1362 qp
->qplib_qp
.modify_flags
|=
1363 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE
|
1364 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE
|
1365 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE
|
1366 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE
|
1367 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA
;
1368 if ((qp_attr
->cap
.max_send_wr
>= dev_attr
->max_qp_wqes
) ||
1369 (qp_attr
->cap
.max_recv_wr
>= dev_attr
->max_qp_wqes
) ||
1370 (qp_attr
->cap
.max_send_sge
>= dev_attr
->max_qp_sges
) ||
1371 (qp_attr
->cap
.max_recv_sge
>= dev_attr
->max_qp_sges
) ||
1372 (qp_attr
->cap
.max_inline_data
>=
1373 dev_attr
->max_inline_data
)) {
1374 dev_err(rdev_to_dev(rdev
),
1375 "Create QP failed - max exceeded");
1378 entries
= roundup_pow_of_two(qp_attr
->cap
.max_send_wr
);
1379 qp
->qplib_qp
.sq
.max_wqe
= min_t(u32
, entries
,
1380 dev_attr
->max_qp_wqes
+ 1);
1381 qp
->qplib_qp
.sq
.max_sge
= qp_attr
->cap
.max_send_sge
;
1382 if (qp
->qplib_qp
.rq
.max_wqe
) {
1383 entries
= roundup_pow_of_two(qp_attr
->cap
.max_recv_wr
);
1384 qp
->qplib_qp
.rq
.max_wqe
=
1385 min_t(u32
, entries
, dev_attr
->max_qp_wqes
+ 1);
1386 qp
->qplib_qp
.rq
.max_sge
= qp_attr
->cap
.max_recv_sge
;
1388 /* SRQ was used prior, just ignore the RQ caps */
1391 if (qp_attr_mask
& IB_QP_DEST_QPN
) {
1392 qp
->qplib_qp
.modify_flags
|=
1393 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID
;
1394 qp
->qplib_qp
.dest_qpn
= qp_attr
->dest_qp_num
;
1396 rc
= bnxt_qplib_modify_qp(&rdev
->qplib_res
, &qp
->qplib_qp
);
1398 dev_err(rdev_to_dev(rdev
), "Failed to modify HW QP");
1401 if (ib_qp
->qp_type
== IB_QPT_GSI
&& rdev
->qp1_sqp
)
1402 rc
= bnxt_re_modify_shadow_qp(rdev
, qp
, qp_attr_mask
);
1406 int bnxt_re_query_qp(struct ib_qp
*ib_qp
, struct ib_qp_attr
*qp_attr
,
1407 int qp_attr_mask
, struct ib_qp_init_attr
*qp_init_attr
)
1409 struct bnxt_re_qp
*qp
= container_of(ib_qp
, struct bnxt_re_qp
, ib_qp
);
1410 struct bnxt_re_dev
*rdev
= qp
->rdev
;
1411 struct bnxt_qplib_qp qplib_qp
;
1414 memset(&qplib_qp
, 0, sizeof(struct bnxt_qplib_qp
));
1415 qplib_qp
.id
= qp
->qplib_qp
.id
;
1416 qplib_qp
.ah
.host_sgid_index
= qp
->qplib_qp
.ah
.host_sgid_index
;
1418 rc
= bnxt_qplib_query_qp(&rdev
->qplib_res
, &qplib_qp
);
1420 dev_err(rdev_to_dev(rdev
), "Failed to query HW QP");
1423 qp_attr
->qp_state
= __to_ib_qp_state(qplib_qp
.state
);
1424 qp_attr
->en_sqd_async_notify
= qplib_qp
.en_sqd_async_notify
? 1 : 0;
1425 qp_attr
->qp_access_flags
= __to_ib_access_flags(qplib_qp
.access
);
1426 qp_attr
->pkey_index
= qplib_qp
.pkey_index
;
1427 qp_attr
->qkey
= qplib_qp
.qkey
;
1428 qp_attr
->ah_attr
.type
= RDMA_AH_ATTR_TYPE_ROCE
;
1429 rdma_ah_set_grh(&qp_attr
->ah_attr
, NULL
, qplib_qp
.ah
.flow_label
,
1430 qplib_qp
.ah
.host_sgid_index
,
1431 qplib_qp
.ah
.hop_limit
,
1432 qplib_qp
.ah
.traffic_class
);
1433 rdma_ah_set_dgid_raw(&qp_attr
->ah_attr
, qplib_qp
.ah
.dgid
.data
);
1434 rdma_ah_set_sl(&qp_attr
->ah_attr
, qplib_qp
.ah
.sl
);
1435 ether_addr_copy(qp_attr
->ah_attr
.roce
.dmac
, qplib_qp
.ah
.dmac
);
1436 qp_attr
->path_mtu
= __to_ib_mtu(qplib_qp
.path_mtu
);
1437 qp_attr
->timeout
= qplib_qp
.timeout
;
1438 qp_attr
->retry_cnt
= qplib_qp
.retry_cnt
;
1439 qp_attr
->rnr_retry
= qplib_qp
.rnr_retry
;
1440 qp_attr
->min_rnr_timer
= qplib_qp
.min_rnr_timer
;
1441 qp_attr
->rq_psn
= qplib_qp
.rq
.psn
;
1442 qp_attr
->max_rd_atomic
= qplib_qp
.max_rd_atomic
;
1443 qp_attr
->sq_psn
= qplib_qp
.sq
.psn
;
1444 qp_attr
->max_dest_rd_atomic
= qplib_qp
.max_dest_rd_atomic
;
1445 qp_init_attr
->sq_sig_type
= qplib_qp
.sig_type
? IB_SIGNAL_ALL_WR
:
1447 qp_attr
->dest_qp_num
= qplib_qp
.dest_qpn
;
1449 qp_attr
->cap
.max_send_wr
= qp
->qplib_qp
.sq
.max_wqe
;
1450 qp_attr
->cap
.max_send_sge
= qp
->qplib_qp
.sq
.max_sge
;
1451 qp_attr
->cap
.max_recv_wr
= qp
->qplib_qp
.rq
.max_wqe
;
1452 qp_attr
->cap
.max_recv_sge
= qp
->qplib_qp
.rq
.max_sge
;
1453 qp_attr
->cap
.max_inline_data
= qp
->qplib_qp
.max_inline_data
;
1454 qp_init_attr
->cap
= qp_attr
->cap
;
1459 /* Routine for sending QP1 packets for RoCE V1 an V2
1461 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp
*qp
,
1462 struct ib_send_wr
*wr
,
1463 struct bnxt_qplib_swqe
*wqe
,
1466 struct ib_device
*ibdev
= &qp
->rdev
->ibdev
;
1467 struct bnxt_re_ah
*ah
= container_of(ud_wr(wr
)->ah
, struct bnxt_re_ah
,
1469 struct bnxt_qplib_ah
*qplib_ah
= &ah
->qplib_ah
;
1470 struct bnxt_qplib_sge sge
;
1474 struct ib_gid_attr sgid_attr
;
1476 bool is_eth
= false;
1477 bool is_vlan
= false;
1478 bool is_grh
= false;
1479 bool is_udp
= false;
1481 u16 vlan_id
= 0xFFFF;
1483 int i
, rc
= 0, size
;
1485 memset(&qp
->qp1_hdr
, 0, sizeof(qp
->qp1_hdr
));
1487 rc
= ib_get_cached_gid(ibdev
, 1,
1488 qplib_ah
->host_sgid_index
, &sgid
,
1491 dev_err(rdev_to_dev(qp
->rdev
),
1492 "Failed to query gid at index %d",
1493 qplib_ah
->host_sgid_index
);
1496 if (sgid_attr
.ndev
) {
1497 if (is_vlan_dev(sgid_attr
.ndev
))
1498 vlan_id
= vlan_dev_vlan_id(sgid_attr
.ndev
);
1499 dev_put(sgid_attr
.ndev
);
1501 /* Get network header type for this GID */
1502 nw_type
= ib_gid_to_network_type(sgid_attr
.gid_type
, &sgid
);
1504 case RDMA_NETWORK_IPV4
:
1505 nw_type
= BNXT_RE_ROCEV2_IPV4_PACKET
;
1507 case RDMA_NETWORK_IPV6
:
1508 nw_type
= BNXT_RE_ROCEV2_IPV6_PACKET
;
1511 nw_type
= BNXT_RE_ROCE_V1_PACKET
;
1514 memcpy(&dgid
.raw
, &qplib_ah
->dgid
, 16);
1515 is_udp
= sgid_attr
.gid_type
== IB_GID_TYPE_ROCE_UDP_ENCAP
;
1517 if (ipv6_addr_v4mapped((struct in6_addr
*)&sgid
)) {
1519 ether_type
= ETH_P_IP
;
1522 ether_type
= ETH_P_IPV6
;
1526 ether_type
= ETH_P_IBOE
;
1531 is_vlan
= (vlan_id
&& (vlan_id
< 0x1000)) ? true : false;
1533 ib_ud_header_init(payload_size
, !is_eth
, is_eth
, is_vlan
, is_grh
,
1534 ip_version
, is_udp
, 0, &qp
->qp1_hdr
);
1537 ether_addr_copy(qp
->qp1_hdr
.eth
.dmac_h
, ah
->qplib_ah
.dmac
);
1538 ether_addr_copy(qp
->qp1_hdr
.eth
.smac_h
, qp
->qplib_qp
.smac
);
1540 /* For vlan, check the sgid for vlan existence */
1543 qp
->qp1_hdr
.eth
.type
= cpu_to_be16(ether_type
);
1545 qp
->qp1_hdr
.vlan
.type
= cpu_to_be16(ether_type
);
1546 qp
->qp1_hdr
.vlan
.tag
= cpu_to_be16(vlan_id
);
1549 if (is_grh
|| (ip_version
== 6)) {
1550 memcpy(qp
->qp1_hdr
.grh
.source_gid
.raw
, sgid
.raw
, sizeof(sgid
));
1551 memcpy(qp
->qp1_hdr
.grh
.destination_gid
.raw
, qplib_ah
->dgid
.data
,
1553 qp
->qp1_hdr
.grh
.hop_limit
= qplib_ah
->hop_limit
;
1556 if (ip_version
== 4) {
1557 qp
->qp1_hdr
.ip4
.tos
= 0;
1558 qp
->qp1_hdr
.ip4
.id
= 0;
1559 qp
->qp1_hdr
.ip4
.frag_off
= htons(IP_DF
);
1560 qp
->qp1_hdr
.ip4
.ttl
= qplib_ah
->hop_limit
;
1562 memcpy(&qp
->qp1_hdr
.ip4
.saddr
, sgid
.raw
+ 12, 4);
1563 memcpy(&qp
->qp1_hdr
.ip4
.daddr
, qplib_ah
->dgid
.data
+ 12, 4);
1564 qp
->qp1_hdr
.ip4
.check
= ib_ud_ip4_csum(&qp
->qp1_hdr
);
1568 qp
->qp1_hdr
.udp
.dport
= htons(ROCE_V2_UDP_DPORT
);
1569 qp
->qp1_hdr
.udp
.sport
= htons(0x8CD1);
1570 qp
->qp1_hdr
.udp
.csum
= 0;
1574 if (wr
->opcode
== IB_WR_SEND_WITH_IMM
) {
1575 qp
->qp1_hdr
.bth
.opcode
= IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE
;
1576 qp
->qp1_hdr
.immediate_present
= 1;
1578 qp
->qp1_hdr
.bth
.opcode
= IB_OPCODE_UD_SEND_ONLY
;
1580 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1581 qp
->qp1_hdr
.bth
.solicited_event
= 1;
1583 qp
->qp1_hdr
.bth
.pad_count
= (4 - payload_size
) & 3;
1585 /* P_key for QP1 is for all members */
1586 qp
->qp1_hdr
.bth
.pkey
= cpu_to_be16(0xFFFF);
1587 qp
->qp1_hdr
.bth
.destination_qpn
= IB_QP1
;
1588 qp
->qp1_hdr
.bth
.ack_req
= 0;
1590 qp
->send_psn
&= BTH_PSN_MASK
;
1591 qp
->qp1_hdr
.bth
.psn
= cpu_to_be32(qp
->send_psn
);
1593 /* Use the priviledged Q_Key for QP1 */
1594 qp
->qp1_hdr
.deth
.qkey
= cpu_to_be32(IB_QP1_QKEY
);
1595 qp
->qp1_hdr
.deth
.source_qpn
= IB_QP1
;
1597 /* Pack the QP1 to the transmit buffer */
1598 buf
= bnxt_qplib_get_qp1_sq_buf(&qp
->qplib_qp
, &sge
);
1600 size
= ib_ud_header_pack(&qp
->qp1_hdr
, buf
);
1601 for (i
= wqe
->num_sge
; i
; i
--) {
1602 wqe
->sg_list
[i
].addr
= wqe
->sg_list
[i
- 1].addr
;
1603 wqe
->sg_list
[i
].lkey
= wqe
->sg_list
[i
- 1].lkey
;
1604 wqe
->sg_list
[i
].size
= wqe
->sg_list
[i
- 1].size
;
1608 * Max Header buf size for IPV6 RoCE V2 is 86,
1609 * which is same as the QP1 SQ header buffer.
1610 * Header buf size for IPV4 RoCE V2 can be 66.
1611 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
1612 * Subtract 20 bytes from QP1 SQ header buf size
1614 if (is_udp
&& ip_version
== 4)
1617 * Max Header buf size for RoCE V1 is 78.
1618 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
1619 * Subtract 8 bytes from QP1 SQ header buf size
1624 /* Subtract 4 bytes for non vlan packets */
1628 wqe
->sg_list
[0].addr
= sge
.addr
;
1629 wqe
->sg_list
[0].lkey
= sge
.lkey
;
1630 wqe
->sg_list
[0].size
= sge
.size
;
1634 dev_err(rdev_to_dev(qp
->rdev
), "QP1 buffer is empty!");
1640 /* For the MAD layer, it only provides the recv SGE the size of
1641 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
1642 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
1643 * receive packet (334 bytes) with no VLAN and then copy the GRH
1644 * and the MAD datagram out to the provided SGE.
1646 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp
*qp
,
1647 struct ib_recv_wr
*wr
,
1648 struct bnxt_qplib_swqe
*wqe
,
1651 struct bnxt_qplib_sge ref
, sge
;
1653 struct bnxt_re_sqp_entries
*sqp_entry
;
1655 rq_prod_index
= bnxt_qplib_get_rq_prod_index(&qp
->qplib_qp
);
1657 if (!bnxt_qplib_get_qp1_rq_buf(&qp
->qplib_qp
, &sge
))
1660 /* Create 1 SGE to receive the entire
1663 /* Save the reference from ULP */
1664 ref
.addr
= wqe
->sg_list
[0].addr
;
1665 ref
.lkey
= wqe
->sg_list
[0].lkey
;
1666 ref
.size
= wqe
->sg_list
[0].size
;
1668 sqp_entry
= &qp
->rdev
->sqp_tbl
[rq_prod_index
];
1671 wqe
->sg_list
[0].addr
= sge
.addr
;
1672 wqe
->sg_list
[0].lkey
= sge
.lkey
;
1673 wqe
->sg_list
[0].size
= BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2
;
1674 sge
.size
-= wqe
->sg_list
[0].size
;
1676 sqp_entry
->sge
.addr
= ref
.addr
;
1677 sqp_entry
->sge
.lkey
= ref
.lkey
;
1678 sqp_entry
->sge
.size
= ref
.size
;
1679 /* Store the wrid for reporting completion */
1680 sqp_entry
->wrid
= wqe
->wr_id
;
1681 /* change the wqe->wrid to table index */
1682 wqe
->wr_id
= rq_prod_index
;
1686 static int is_ud_qp(struct bnxt_re_qp
*qp
)
1688 return qp
->qplib_qp
.type
== CMDQ_CREATE_QP_TYPE_UD
;
1691 static int bnxt_re_build_send_wqe(struct bnxt_re_qp
*qp
,
1692 struct ib_send_wr
*wr
,
1693 struct bnxt_qplib_swqe
*wqe
)
1695 struct bnxt_re_ah
*ah
= NULL
;
1698 ah
= container_of(ud_wr(wr
)->ah
, struct bnxt_re_ah
, ib_ah
);
1699 wqe
->send
.q_key
= ud_wr(wr
)->remote_qkey
;
1700 wqe
->send
.dst_qp
= ud_wr(wr
)->remote_qpn
;
1701 wqe
->send
.avid
= ah
->qplib_ah
.id
;
1703 switch (wr
->opcode
) {
1705 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_SEND
;
1707 case IB_WR_SEND_WITH_IMM
:
1708 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM
;
1709 wqe
->send
.imm_data
= wr
->ex
.imm_data
;
1711 case IB_WR_SEND_WITH_INV
:
1712 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV
;
1713 wqe
->send
.inv_key
= wr
->ex
.invalidate_rkey
;
1718 if (wr
->send_flags
& IB_SEND_SIGNALED
)
1719 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP
;
1720 if (wr
->send_flags
& IB_SEND_FENCE
)
1721 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE
;
1722 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1723 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT
;
1724 if (wr
->send_flags
& IB_SEND_INLINE
)
1725 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_INLINE
;
1730 static int bnxt_re_build_rdma_wqe(struct ib_send_wr
*wr
,
1731 struct bnxt_qplib_swqe
*wqe
)
1733 switch (wr
->opcode
) {
1734 case IB_WR_RDMA_WRITE
:
1735 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE
;
1737 case IB_WR_RDMA_WRITE_WITH_IMM
:
1738 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM
;
1739 wqe
->rdma
.imm_data
= wr
->ex
.imm_data
;
1741 case IB_WR_RDMA_READ
:
1742 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_RDMA_READ
;
1743 wqe
->rdma
.inv_key
= wr
->ex
.invalidate_rkey
;
1748 wqe
->rdma
.remote_va
= rdma_wr(wr
)->remote_addr
;
1749 wqe
->rdma
.r_key
= rdma_wr(wr
)->rkey
;
1750 if (wr
->send_flags
& IB_SEND_SIGNALED
)
1751 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP
;
1752 if (wr
->send_flags
& IB_SEND_FENCE
)
1753 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE
;
1754 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1755 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT
;
1756 if (wr
->send_flags
& IB_SEND_INLINE
)
1757 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_INLINE
;
1762 static int bnxt_re_build_atomic_wqe(struct ib_send_wr
*wr
,
1763 struct bnxt_qplib_swqe
*wqe
)
1765 switch (wr
->opcode
) {
1766 case IB_WR_ATOMIC_CMP_AND_SWP
:
1767 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP
;
1768 wqe
->atomic
.swap_data
= atomic_wr(wr
)->swap
;
1770 case IB_WR_ATOMIC_FETCH_AND_ADD
:
1771 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD
;
1772 wqe
->atomic
.cmp_data
= atomic_wr(wr
)->compare_add
;
1777 wqe
->atomic
.remote_va
= atomic_wr(wr
)->remote_addr
;
1778 wqe
->atomic
.r_key
= atomic_wr(wr
)->rkey
;
1779 if (wr
->send_flags
& IB_SEND_SIGNALED
)
1780 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP
;
1781 if (wr
->send_flags
& IB_SEND_FENCE
)
1782 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE
;
1783 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1784 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT
;
1788 static int bnxt_re_build_inv_wqe(struct ib_send_wr
*wr
,
1789 struct bnxt_qplib_swqe
*wqe
)
1791 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_LOCAL_INV
;
1792 wqe
->local_inv
.inv_l_key
= wr
->ex
.invalidate_rkey
;
1794 if (wr
->send_flags
& IB_SEND_SIGNALED
)
1795 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP
;
1796 if (wr
->send_flags
& IB_SEND_FENCE
)
1797 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE
;
1798 if (wr
->send_flags
& IB_SEND_SOLICITED
)
1799 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT
;
1804 static int bnxt_re_build_reg_wqe(struct ib_reg_wr
*wr
,
1805 struct bnxt_qplib_swqe
*wqe
)
1807 struct bnxt_re_mr
*mr
= container_of(wr
->mr
, struct bnxt_re_mr
, ib_mr
);
1808 struct bnxt_qplib_frpl
*qplib_frpl
= &mr
->qplib_frpl
;
1809 int access
= wr
->access
;
1811 wqe
->frmr
.pbl_ptr
= (__le64
*)qplib_frpl
->hwq
.pbl_ptr
[0];
1812 wqe
->frmr
.pbl_dma_ptr
= qplib_frpl
->hwq
.pbl_dma_ptr
[0];
1813 wqe
->frmr
.page_list
= mr
->pages
;
1814 wqe
->frmr
.page_list_len
= mr
->npages
;
1815 wqe
->frmr
.levels
= qplib_frpl
->hwq
.level
+ 1;
1816 wqe
->type
= BNXT_QPLIB_SWQE_TYPE_REG_MR
;
1818 if (wr
->wr
.send_flags
& IB_SEND_FENCE
)
1819 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE
;
1820 if (wr
->wr
.send_flags
& IB_SEND_SIGNALED
)
1821 wqe
->flags
|= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP
;
1823 if (access
& IB_ACCESS_LOCAL_WRITE
)
1824 wqe
->frmr
.access_cntl
|= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE
;
1825 if (access
& IB_ACCESS_REMOTE_READ
)
1826 wqe
->frmr
.access_cntl
|= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ
;
1827 if (access
& IB_ACCESS_REMOTE_WRITE
)
1828 wqe
->frmr
.access_cntl
|= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE
;
1829 if (access
& IB_ACCESS_REMOTE_ATOMIC
)
1830 wqe
->frmr
.access_cntl
|= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC
;
1831 if (access
& IB_ACCESS_MW_BIND
)
1832 wqe
->frmr
.access_cntl
|= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND
;
1834 wqe
->frmr
.l_key
= wr
->key
;
1835 wqe
->frmr
.length
= wr
->mr
->length
;
1836 wqe
->frmr
.pbl_pg_sz_log
= (wr
->mr
->page_size
>> PAGE_SHIFT_4K
) - 1;
1837 wqe
->frmr
.va
= wr
->mr
->iova
;
1841 static int bnxt_re_copy_inline_data(struct bnxt_re_dev
*rdev
,
1842 struct ib_send_wr
*wr
,
1843 struct bnxt_qplib_swqe
*wqe
)
1845 /* Copy the inline data to the data field */
1850 in_data
= wqe
->inline_data
;
1851 for (i
= 0; i
< wr
->num_sge
; i
++) {
1852 sge_addr
= (void *)(unsigned long)
1853 wr
->sg_list
[i
].addr
;
1854 sge_len
= wr
->sg_list
[i
].length
;
1856 if ((sge_len
+ wqe
->inline_len
) >
1857 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH
) {
1858 dev_err(rdev_to_dev(rdev
),
1859 "Inline data size requested > supported value");
1862 sge_len
= wr
->sg_list
[i
].length
;
1864 memcpy(in_data
, sge_addr
, sge_len
);
1865 in_data
+= wr
->sg_list
[i
].length
;
1866 wqe
->inline_len
+= wr
->sg_list
[i
].length
;
1868 return wqe
->inline_len
;
1871 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev
*rdev
,
1872 struct ib_send_wr
*wr
,
1873 struct bnxt_qplib_swqe
*wqe
)
1877 if (wr
->send_flags
& IB_SEND_INLINE
)
1878 payload_sz
= bnxt_re_copy_inline_data(rdev
, wr
, wqe
);
1880 payload_sz
= bnxt_re_build_sgl(wr
->sg_list
, wqe
->sg_list
,
1886 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev
*rdev
,
1887 struct bnxt_re_qp
*qp
,
1888 struct ib_send_wr
*wr
)
1890 struct bnxt_qplib_swqe wqe
;
1891 int rc
= 0, payload_sz
= 0;
1892 unsigned long flags
;
1894 spin_lock_irqsave(&qp
->sq_lock
, flags
);
1895 memset(&wqe
, 0, sizeof(wqe
));
1898 memset(&wqe
, 0, sizeof(wqe
));
1901 wqe
.num_sge
= wr
->num_sge
;
1902 if (wr
->num_sge
> qp
->qplib_qp
.sq
.max_sge
) {
1903 dev_err(rdev_to_dev(rdev
),
1904 "Limit exceeded for Send SGEs");
1909 payload_sz
= bnxt_re_copy_wr_payload(qp
->rdev
, wr
, &wqe
);
1910 if (payload_sz
< 0) {
1914 wqe
.wr_id
= wr
->wr_id
;
1916 wqe
.type
= BNXT_QPLIB_SWQE_TYPE_SEND
;
1918 rc
= bnxt_re_build_send_wqe(qp
, wr
, &wqe
);
1920 rc
= bnxt_qplib_post_send(&qp
->qplib_qp
, &wqe
);
1923 dev_err(rdev_to_dev(rdev
),
1924 "Post send failed opcode = %#x rc = %d",
1930 bnxt_qplib_post_send_db(&qp
->qplib_qp
);
1931 spin_unlock_irqrestore(&qp
->sq_lock
, flags
);
1935 int bnxt_re_post_send(struct ib_qp
*ib_qp
, struct ib_send_wr
*wr
,
1936 struct ib_send_wr
**bad_wr
)
1938 struct bnxt_re_qp
*qp
= container_of(ib_qp
, struct bnxt_re_qp
, ib_qp
);
1939 struct bnxt_qplib_swqe wqe
;
1940 int rc
= 0, payload_sz
= 0;
1941 unsigned long flags
;
1943 spin_lock_irqsave(&qp
->sq_lock
, flags
);
1946 memset(&wqe
, 0, sizeof(wqe
));
1949 wqe
.num_sge
= wr
->num_sge
;
1950 if (wr
->num_sge
> qp
->qplib_qp
.sq
.max_sge
) {
1951 dev_err(rdev_to_dev(qp
->rdev
),
1952 "Limit exceeded for Send SGEs");
1957 payload_sz
= bnxt_re_copy_wr_payload(qp
->rdev
, wr
, &wqe
);
1958 if (payload_sz
< 0) {
1962 wqe
.wr_id
= wr
->wr_id
;
1964 switch (wr
->opcode
) {
1966 case IB_WR_SEND_WITH_IMM
:
1967 if (ib_qp
->qp_type
== IB_QPT_GSI
) {
1968 rc
= bnxt_re_build_qp1_send_v2(qp
, wr
, &wqe
,
1972 wqe
.rawqp1
.lflags
|=
1973 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC
;
1975 switch (wr
->send_flags
) {
1976 case IB_SEND_IP_CSUM
:
1977 wqe
.rawqp1
.lflags
|=
1978 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM
;
1983 /* Fall thru to build the wqe */
1984 case IB_WR_SEND_WITH_INV
:
1985 rc
= bnxt_re_build_send_wqe(qp
, wr
, &wqe
);
1987 case IB_WR_RDMA_WRITE
:
1988 case IB_WR_RDMA_WRITE_WITH_IMM
:
1989 case IB_WR_RDMA_READ
:
1990 rc
= bnxt_re_build_rdma_wqe(wr
, &wqe
);
1992 case IB_WR_ATOMIC_CMP_AND_SWP
:
1993 case IB_WR_ATOMIC_FETCH_AND_ADD
:
1994 rc
= bnxt_re_build_atomic_wqe(wr
, &wqe
);
1996 case IB_WR_RDMA_READ_WITH_INV
:
1997 dev_err(rdev_to_dev(qp
->rdev
),
1998 "RDMA Read with Invalidate is not supported");
2001 case IB_WR_LOCAL_INV
:
2002 rc
= bnxt_re_build_inv_wqe(wr
, &wqe
);
2005 rc
= bnxt_re_build_reg_wqe(reg_wr(wr
), &wqe
);
2008 /* Unsupported WRs */
2009 dev_err(rdev_to_dev(qp
->rdev
),
2010 "WR (%#x) is not supported", wr
->opcode
);
2015 rc
= bnxt_qplib_post_send(&qp
->qplib_qp
, &wqe
);
2018 dev_err(rdev_to_dev(qp
->rdev
),
2019 "post_send failed op:%#x qps = %#x rc = %d\n",
2020 wr
->opcode
, qp
->qplib_qp
.state
, rc
);
2026 bnxt_qplib_post_send_db(&qp
->qplib_qp
);
2027 spin_unlock_irqrestore(&qp
->sq_lock
, flags
);
2032 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev
*rdev
,
2033 struct bnxt_re_qp
*qp
,
2034 struct ib_recv_wr
*wr
)
2036 struct bnxt_qplib_swqe wqe
;
2037 int rc
= 0, payload_sz
= 0;
2039 memset(&wqe
, 0, sizeof(wqe
));
2042 memset(&wqe
, 0, sizeof(wqe
));
2045 wqe
.num_sge
= wr
->num_sge
;
2046 if (wr
->num_sge
> qp
->qplib_qp
.rq
.max_sge
) {
2047 dev_err(rdev_to_dev(rdev
),
2048 "Limit exceeded for Receive SGEs");
2052 payload_sz
= bnxt_re_build_sgl(wr
->sg_list
, wqe
.sg_list
,
2054 wqe
.wr_id
= wr
->wr_id
;
2055 wqe
.type
= BNXT_QPLIB_SWQE_TYPE_RECV
;
2057 rc
= bnxt_qplib_post_recv(&qp
->qplib_qp
, &wqe
);
2064 bnxt_qplib_post_recv_db(&qp
->qplib_qp
);
2068 int bnxt_re_post_recv(struct ib_qp
*ib_qp
, struct ib_recv_wr
*wr
,
2069 struct ib_recv_wr
**bad_wr
)
2071 struct bnxt_re_qp
*qp
= container_of(ib_qp
, struct bnxt_re_qp
, ib_qp
);
2072 struct bnxt_qplib_swqe wqe
;
2073 int rc
= 0, payload_sz
= 0;
2077 memset(&wqe
, 0, sizeof(wqe
));
2080 wqe
.num_sge
= wr
->num_sge
;
2081 if (wr
->num_sge
> qp
->qplib_qp
.rq
.max_sge
) {
2082 dev_err(rdev_to_dev(qp
->rdev
),
2083 "Limit exceeded for Receive SGEs");
2089 payload_sz
= bnxt_re_build_sgl(wr
->sg_list
, wqe
.sg_list
,
2091 wqe
.wr_id
= wr
->wr_id
;
2092 wqe
.type
= BNXT_QPLIB_SWQE_TYPE_RECV
;
2094 if (ib_qp
->qp_type
== IB_QPT_GSI
)
2095 rc
= bnxt_re_build_qp1_shadow_qp_recv(qp
, wr
, &wqe
,
2098 rc
= bnxt_qplib_post_recv(&qp
->qplib_qp
, &wqe
);
2105 bnxt_qplib_post_recv_db(&qp
->qplib_qp
);
2109 /* Completion Queues */
2110 int bnxt_re_destroy_cq(struct ib_cq
*ib_cq
)
2112 struct bnxt_re_cq
*cq
= container_of(ib_cq
, struct bnxt_re_cq
, ib_cq
);
2113 struct bnxt_re_dev
*rdev
= cq
->rdev
;
2116 rc
= bnxt_qplib_destroy_cq(&rdev
->qplib_res
, &cq
->qplib_cq
);
2118 dev_err(rdev_to_dev(rdev
), "Failed to destroy HW CQ");
2121 if (!IS_ERR_OR_NULL(cq
->umem
))
2122 ib_umem_release(cq
->umem
);
2128 atomic_dec(&rdev
->cq_count
);
2133 struct ib_cq
*bnxt_re_create_cq(struct ib_device
*ibdev
,
2134 const struct ib_cq_init_attr
*attr
,
2135 struct ib_ucontext
*context
,
2136 struct ib_udata
*udata
)
2138 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
2139 struct bnxt_qplib_dev_attr
*dev_attr
= &rdev
->dev_attr
;
2140 struct bnxt_re_cq
*cq
= NULL
;
2142 int cqe
= attr
->cqe
;
2144 /* Validate CQ fields */
2145 if (cqe
< 1 || cqe
> dev_attr
->max_cq_wqes
) {
2146 dev_err(rdev_to_dev(rdev
), "Failed to create CQ -max exceeded");
2147 return ERR_PTR(-EINVAL
);
2149 cq
= kzalloc(sizeof(*cq
), GFP_KERNEL
);
2151 return ERR_PTR(-ENOMEM
);
2154 cq
->qplib_cq
.cq_handle
= (u64
)(unsigned long)(&cq
->qplib_cq
);
2156 entries
= roundup_pow_of_two(cqe
+ 1);
2157 if (entries
> dev_attr
->max_cq_wqes
+ 1)
2158 entries
= dev_attr
->max_cq_wqes
+ 1;
2161 struct bnxt_re_cq_req req
;
2162 struct bnxt_re_ucontext
*uctx
= container_of
2164 struct bnxt_re_ucontext
,
2166 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
2171 cq
->umem
= ib_umem_get(context
, req
.cq_va
,
2172 entries
* sizeof(struct cq_base
),
2173 IB_ACCESS_LOCAL_WRITE
, 1);
2174 if (IS_ERR(cq
->umem
)) {
2175 rc
= PTR_ERR(cq
->umem
);
2178 cq
->qplib_cq
.sghead
= cq
->umem
->sg_head
.sgl
;
2179 cq
->qplib_cq
.nmap
= cq
->umem
->nmap
;
2180 cq
->qplib_cq
.dpi
= uctx
->dpi
;
2182 cq
->max_cql
= min_t(u32
, entries
, MAX_CQL_PER_POLL
);
2183 cq
->cql
= kcalloc(cq
->max_cql
, sizeof(struct bnxt_qplib_cqe
),
2190 cq
->qplib_cq
.dpi
= &rdev
->dpi_privileged
;
2191 cq
->qplib_cq
.sghead
= NULL
;
2192 cq
->qplib_cq
.nmap
= 0;
2194 cq
->qplib_cq
.max_wqe
= entries
;
2195 cq
->qplib_cq
.cnq_hw_ring_id
= rdev
->nq
.ring_id
;
2197 rc
= bnxt_qplib_create_cq(&rdev
->qplib_res
, &cq
->qplib_cq
);
2199 dev_err(rdev_to_dev(rdev
), "Failed to create HW CQ");
2203 cq
->ib_cq
.cqe
= entries
;
2204 cq
->cq_period
= cq
->qplib_cq
.period
;
2207 atomic_inc(&rdev
->cq_count
);
2210 struct bnxt_re_cq_resp resp
;
2212 resp
.cqid
= cq
->qplib_cq
.id
;
2213 resp
.tail
= cq
->qplib_cq
.hwq
.cons
;
2214 resp
.phase
= cq
->qplib_cq
.period
;
2216 rc
= ib_copy_to_udata(udata
, &resp
, sizeof(resp
));
2218 dev_err(rdev_to_dev(rdev
), "Failed to copy CQ udata");
2219 bnxt_qplib_destroy_cq(&rdev
->qplib_res
, &cq
->qplib_cq
);
2228 ib_umem_release(cq
->umem
);
2235 static u8
__req_to_ib_wc_status(u8 qstatus
)
2238 case CQ_REQ_STATUS_OK
:
2239 return IB_WC_SUCCESS
;
2240 case CQ_REQ_STATUS_BAD_RESPONSE_ERR
:
2241 return IB_WC_BAD_RESP_ERR
;
2242 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR
:
2243 return IB_WC_LOC_LEN_ERR
;
2244 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR
:
2245 return IB_WC_LOC_QP_OP_ERR
;
2246 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR
:
2247 return IB_WC_LOC_PROT_ERR
;
2248 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR
:
2249 return IB_WC_GENERAL_ERR
;
2250 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR
:
2251 return IB_WC_REM_INV_REQ_ERR
;
2252 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR
:
2253 return IB_WC_REM_ACCESS_ERR
;
2254 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR
:
2255 return IB_WC_REM_OP_ERR
;
2256 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR
:
2257 return IB_WC_RNR_RETRY_EXC_ERR
;
2258 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR
:
2259 return IB_WC_RETRY_EXC_ERR
;
2260 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR
:
2261 return IB_WC_WR_FLUSH_ERR
;
2263 return IB_WC_GENERAL_ERR
;
2268 static u8
__rawqp1_to_ib_wc_status(u8 qstatus
)
2271 case CQ_RES_RAWETH_QP1_STATUS_OK
:
2272 return IB_WC_SUCCESS
;
2273 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR
:
2274 return IB_WC_LOC_ACCESS_ERR
;
2275 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR
:
2276 return IB_WC_LOC_LEN_ERR
;
2277 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR
:
2278 return IB_WC_LOC_PROT_ERR
;
2279 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR
:
2280 return IB_WC_LOC_QP_OP_ERR
;
2281 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR
:
2282 return IB_WC_GENERAL_ERR
;
2283 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR
:
2284 return IB_WC_WR_FLUSH_ERR
;
2285 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR
:
2286 return IB_WC_WR_FLUSH_ERR
;
2288 return IB_WC_GENERAL_ERR
;
2292 static u8
__rc_to_ib_wc_status(u8 qstatus
)
2295 case CQ_RES_RC_STATUS_OK
:
2296 return IB_WC_SUCCESS
;
2297 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR
:
2298 return IB_WC_LOC_ACCESS_ERR
;
2299 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR
:
2300 return IB_WC_LOC_LEN_ERR
;
2301 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR
:
2302 return IB_WC_LOC_PROT_ERR
;
2303 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR
:
2304 return IB_WC_LOC_QP_OP_ERR
;
2305 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR
:
2306 return IB_WC_GENERAL_ERR
;
2307 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR
:
2308 return IB_WC_REM_INV_REQ_ERR
;
2309 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR
:
2310 return IB_WC_WR_FLUSH_ERR
;
2311 case CQ_RES_RC_STATUS_HW_FLUSH_ERR
:
2312 return IB_WC_WR_FLUSH_ERR
;
2314 return IB_WC_GENERAL_ERR
;
2318 static void bnxt_re_process_req_wc(struct ib_wc
*wc
, struct bnxt_qplib_cqe
*cqe
)
2320 switch (cqe
->type
) {
2321 case BNXT_QPLIB_SWQE_TYPE_SEND
:
2322 wc
->opcode
= IB_WC_SEND
;
2324 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM
:
2325 wc
->opcode
= IB_WC_SEND
;
2326 wc
->wc_flags
|= IB_WC_WITH_IMM
;
2328 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV
:
2329 wc
->opcode
= IB_WC_SEND
;
2330 wc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
2332 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE
:
2333 wc
->opcode
= IB_WC_RDMA_WRITE
;
2335 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM
:
2336 wc
->opcode
= IB_WC_RDMA_WRITE
;
2337 wc
->wc_flags
|= IB_WC_WITH_IMM
;
2339 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ
:
2340 wc
->opcode
= IB_WC_RDMA_READ
;
2342 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP
:
2343 wc
->opcode
= IB_WC_COMP_SWAP
;
2345 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD
:
2346 wc
->opcode
= IB_WC_FETCH_ADD
;
2348 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV
:
2349 wc
->opcode
= IB_WC_LOCAL_INV
;
2351 case BNXT_QPLIB_SWQE_TYPE_REG_MR
:
2352 wc
->opcode
= IB_WC_REG_MR
;
2355 wc
->opcode
= IB_WC_SEND
;
2359 wc
->status
= __req_to_ib_wc_status(cqe
->status
);
2362 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags
,
2363 u16 raweth_qp1_flags2
)
2365 bool is_udp
= false, is_ipv6
= false, is_ipv4
= false;
2367 /* raweth_qp1_flags Bit 9-6 indicates itype */
2368 if ((raweth_qp1_flags
& CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE
)
2369 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE
)
2372 if (raweth_qp1_flags2
&
2373 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC
&&
2375 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC
) {
2377 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
2378 (raweth_qp1_flags2
&
2379 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE
) ?
2380 (is_ipv6
= true) : (is_ipv4
= true);
2382 BNXT_RE_ROCEV2_IPV6_PACKET
:
2383 BNXT_RE_ROCEV2_IPV4_PACKET
);
2385 return BNXT_RE_ROCE_V1_PACKET
;
2389 static int bnxt_re_to_ib_nw_type(int nw_type
)
2391 u8 nw_hdr_type
= 0xFF;
2394 case BNXT_RE_ROCE_V1_PACKET
:
2395 nw_hdr_type
= RDMA_NETWORK_ROCE_V1
;
2397 case BNXT_RE_ROCEV2_IPV4_PACKET
:
2398 nw_hdr_type
= RDMA_NETWORK_IPV4
;
2400 case BNXT_RE_ROCEV2_IPV6_PACKET
:
2401 nw_hdr_type
= RDMA_NETWORK_IPV6
;
2407 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev
*rdev
,
2411 struct ethhdr
*eth_hdr
;
2415 tmp_buf
= (u8
*)rq_hdr_buf
;
2417 * If dest mac is not same as I/F mac, this could be a
2418 * loopback address or multicast address, check whether
2419 * it is a loopback packet
2421 if (!ether_addr_equal(tmp_buf
, rdev
->netdev
->dev_addr
)) {
2423 /* Check the ether type */
2424 eth_hdr
= (struct ethhdr
*)tmp_buf
;
2425 eth_type
= ntohs(eth_hdr
->h_proto
);
2433 struct udphdr
*udp_hdr
;
2435 len
= (eth_type
== ETH_P_IP
? sizeof(struct iphdr
) :
2436 sizeof(struct ipv6hdr
));
2437 tmp_buf
+= sizeof(struct ethhdr
) + len
;
2438 udp_hdr
= (struct udphdr
*)tmp_buf
;
2439 if (ntohs(udp_hdr
->dest
) ==
2452 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp
*qp1_qp
,
2453 struct bnxt_qplib_cqe
*cqe
)
2455 struct bnxt_re_dev
*rdev
= qp1_qp
->rdev
;
2456 struct bnxt_re_sqp_entries
*sqp_entry
= NULL
;
2457 struct bnxt_re_qp
*qp
= rdev
->qp1_sqp
;
2458 struct ib_send_wr
*swr
;
2459 struct ib_ud_wr udwr
;
2460 struct ib_recv_wr rwr
;
2464 dma_addr_t rq_hdr_buf_map
;
2465 dma_addr_t shrq_hdr_buf_map
;
2468 struct ib_sge s_sge
[2];
2469 struct ib_sge r_sge
[2];
2472 memset(&udwr
, 0, sizeof(udwr
));
2473 memset(&rwr
, 0, sizeof(rwr
));
2474 memset(&s_sge
, 0, sizeof(s_sge
));
2475 memset(&r_sge
, 0, sizeof(r_sge
));
2478 tbl_idx
= cqe
->wr_id
;
2480 rq_hdr_buf
= qp1_qp
->qplib_qp
.rq_hdr_buf
+
2481 (tbl_idx
* qp1_qp
->qplib_qp
.rq_hdr_buf_size
);
2482 rq_hdr_buf_map
= bnxt_qplib_get_qp_buf_from_index(&qp1_qp
->qplib_qp
,
2485 /* Shadow QP header buffer */
2486 shrq_hdr_buf_map
= bnxt_qplib_get_qp_buf_from_index(&qp
->qplib_qp
,
2488 sqp_entry
= &rdev
->sqp_tbl
[tbl_idx
];
2490 /* Store this cqe */
2491 memcpy(&sqp_entry
->cqe
, cqe
, sizeof(struct bnxt_qplib_cqe
));
2492 sqp_entry
->qp1_qp
= qp1_qp
;
2494 /* Find packet type from the cqe */
2496 pkt_type
= bnxt_re_check_packet_type(cqe
->raweth_qp1_flags
,
2497 cqe
->raweth_qp1_flags2
);
2499 dev_err(rdev_to_dev(rdev
), "Invalid packet\n");
2503 /* Adjust the offset for the user buffer and post in the rq */
2505 if (pkt_type
== BNXT_RE_ROCEV2_IPV4_PACKET
)
2509 * QP1 loopback packet has 4 bytes of internal header before
2510 * ether header. Skip these four bytes.
2512 if (bnxt_re_is_loopback_packet(rdev
, rq_hdr_buf
))
2515 /* First send SGE . Skip the ether header*/
2516 s_sge
[0].addr
= rq_hdr_buf_map
+ BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
2518 s_sge
[0].lkey
= 0xFFFFFFFF;
2519 s_sge
[0].length
= offset
? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4
:
2520 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6
;
2522 /* Second Send SGE */
2523 s_sge
[1].addr
= s_sge
[0].addr
+ s_sge
[0].length
+
2524 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE
;
2525 if (pkt_type
!= BNXT_RE_ROCE_V1_PACKET
)
2527 s_sge
[1].lkey
= 0xFFFFFFFF;
2528 s_sge
[1].length
= 256;
2530 /* First recv SGE */
2532 r_sge
[0].addr
= shrq_hdr_buf_map
;
2533 r_sge
[0].lkey
= 0xFFFFFFFF;
2534 r_sge
[0].length
= 40;
2536 r_sge
[1].addr
= sqp_entry
->sge
.addr
+ offset
;
2537 r_sge
[1].lkey
= sqp_entry
->sge
.lkey
;
2538 r_sge
[1].length
= BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6
+ 256 - offset
;
2540 /* Create receive work request */
2542 rwr
.sg_list
= r_sge
;
2543 rwr
.wr_id
= tbl_idx
;
2546 rc
= bnxt_re_post_recv_shadow_qp(rdev
, qp
, &rwr
);
2548 dev_err(rdev_to_dev(rdev
),
2549 "Failed to post Rx buffers to shadow QP");
2554 swr
->sg_list
= s_sge
;
2555 swr
->wr_id
= tbl_idx
;
2556 swr
->opcode
= IB_WR_SEND
;
2559 udwr
.ah
= &rdev
->sqp_ah
->ib_ah
;
2560 udwr
.remote_qpn
= rdev
->qp1_sqp
->qplib_qp
.id
;
2561 udwr
.remote_qkey
= rdev
->qp1_sqp
->qplib_qp
.qkey
;
2563 /* post data received in the send queue */
2564 rc
= bnxt_re_post_send_shadow_qp(rdev
, qp
, swr
);
2569 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc
*wc
,
2570 struct bnxt_qplib_cqe
*cqe
)
2572 wc
->opcode
= IB_WC_RECV
;
2573 wc
->status
= __rawqp1_to_ib_wc_status(cqe
->status
);
2574 wc
->wc_flags
|= IB_WC_GRH
;
2577 static void bnxt_re_process_res_rc_wc(struct ib_wc
*wc
,
2578 struct bnxt_qplib_cqe
*cqe
)
2580 wc
->opcode
= IB_WC_RECV
;
2581 wc
->status
= __rc_to_ib_wc_status(cqe
->status
);
2583 if (cqe
->flags
& CQ_RES_RC_FLAGS_IMM
)
2584 wc
->wc_flags
|= IB_WC_WITH_IMM
;
2585 if (cqe
->flags
& CQ_RES_RC_FLAGS_INV
)
2586 wc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
2587 if ((cqe
->flags
& (CQ_RES_RC_FLAGS_RDMA
| CQ_RES_RC_FLAGS_IMM
)) ==
2588 (CQ_RES_RC_FLAGS_RDMA
| CQ_RES_RC_FLAGS_IMM
))
2589 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
2592 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp
*qp
,
2594 struct bnxt_qplib_cqe
*cqe
)
2597 struct bnxt_re_dev
*rdev
= qp
->rdev
;
2598 struct bnxt_re_qp
*qp1_qp
= NULL
;
2599 struct bnxt_qplib_cqe
*orig_cqe
= NULL
;
2600 struct bnxt_re_sqp_entries
*sqp_entry
= NULL
;
2603 tbl_idx
= cqe
->wr_id
;
2605 sqp_entry
= &rdev
->sqp_tbl
[tbl_idx
];
2606 qp1_qp
= sqp_entry
->qp1_qp
;
2607 orig_cqe
= &sqp_entry
->cqe
;
2609 wc
->wr_id
= sqp_entry
->wrid
;
2610 wc
->byte_len
= orig_cqe
->length
;
2611 wc
->qp
= &qp1_qp
->ib_qp
;
2613 wc
->ex
.imm_data
= orig_cqe
->immdata
;
2614 wc
->src_qp
= orig_cqe
->src_qp
;
2615 memcpy(wc
->smac
, orig_cqe
->smac
, ETH_ALEN
);
2617 wc
->vendor_err
= orig_cqe
->status
;
2619 wc
->opcode
= IB_WC_RECV
;
2620 wc
->status
= __rawqp1_to_ib_wc_status(orig_cqe
->status
);
2621 wc
->wc_flags
|= IB_WC_GRH
;
2623 nw_type
= bnxt_re_check_packet_type(orig_cqe
->raweth_qp1_flags
,
2624 orig_cqe
->raweth_qp1_flags2
);
2626 wc
->network_hdr_type
= bnxt_re_to_ib_nw_type(nw_type
);
2627 wc
->wc_flags
|= IB_WC_WITH_NETWORK_HDR_TYPE
;
2631 static void bnxt_re_process_res_ud_wc(struct ib_wc
*wc
,
2632 struct bnxt_qplib_cqe
*cqe
)
2634 wc
->opcode
= IB_WC_RECV
;
2635 wc
->status
= __rc_to_ib_wc_status(cqe
->status
);
2637 if (cqe
->flags
& CQ_RES_RC_FLAGS_IMM
)
2638 wc
->wc_flags
|= IB_WC_WITH_IMM
;
2639 if (cqe
->flags
& CQ_RES_RC_FLAGS_INV
)
2640 wc
->wc_flags
|= IB_WC_WITH_INVALIDATE
;
2641 if ((cqe
->flags
& (CQ_RES_RC_FLAGS_RDMA
| CQ_RES_RC_FLAGS_IMM
)) ==
2642 (CQ_RES_RC_FLAGS_RDMA
| CQ_RES_RC_FLAGS_IMM
))
2643 wc
->opcode
= IB_WC_RECV_RDMA_WITH_IMM
;
2646 int bnxt_re_poll_cq(struct ib_cq
*ib_cq
, int num_entries
, struct ib_wc
*wc
)
2648 struct bnxt_re_cq
*cq
= container_of(ib_cq
, struct bnxt_re_cq
, ib_cq
);
2649 struct bnxt_re_qp
*qp
;
2650 struct bnxt_qplib_cqe
*cqe
;
2651 int i
, ncqe
, budget
;
2653 struct bnxt_re_sqp_entries
*sqp_entry
= NULL
;
2654 unsigned long flags
;
2656 spin_lock_irqsave(&cq
->cq_lock
, flags
);
2657 budget
= min_t(u32
, num_entries
, cq
->max_cql
);
2659 dev_err(rdev_to_dev(cq
->rdev
), "POLL CQ : no CQL to use");
2664 ncqe
= bnxt_qplib_poll_cq(&cq
->qplib_cq
, cqe
, budget
);
2668 for (i
= 0; i
< ncqe
; i
++, cqe
++) {
2669 /* Transcribe each qplib_wqe back to ib_wc */
2670 memset(wc
, 0, sizeof(*wc
));
2672 wc
->wr_id
= cqe
->wr_id
;
2673 wc
->byte_len
= cqe
->length
;
2675 ((struct bnxt_qplib_qp
*)
2676 (unsigned long)(cqe
->qp_handle
),
2677 struct bnxt_re_qp
, qplib_qp
);
2679 dev_err(rdev_to_dev(cq
->rdev
),
2680 "POLL CQ : bad QP handle");
2683 wc
->qp
= &qp
->ib_qp
;
2684 wc
->ex
.imm_data
= cqe
->immdata
;
2685 wc
->src_qp
= cqe
->src_qp
;
2686 memcpy(wc
->smac
, cqe
->smac
, ETH_ALEN
);
2688 wc
->vendor_err
= cqe
->status
;
2690 switch (cqe
->opcode
) {
2691 case CQ_BASE_CQE_TYPE_REQ
:
2692 if (qp
->qplib_qp
.id
==
2693 qp
->rdev
->qp1_sqp
->qplib_qp
.id
) {
2694 /* Handle this completion with
2695 * the stored completion
2697 memset(wc
, 0, sizeof(*wc
));
2700 bnxt_re_process_req_wc(wc
, cqe
);
2702 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1
:
2706 rc
= bnxt_re_process_raw_qp_pkt_rx
2709 memset(wc
, 0, sizeof(*wc
));
2714 /* Errors need not be looped back.
2715 * But change the wr_id to the one
2716 * stored in the table
2718 tbl_idx
= cqe
->wr_id
;
2719 sqp_entry
= &cq
->rdev
->sqp_tbl
[tbl_idx
];
2720 wc
->wr_id
= sqp_entry
->wrid
;
2721 bnxt_re_process_res_rawqp1_wc(wc
, cqe
);
2723 case CQ_BASE_CQE_TYPE_RES_RC
:
2724 bnxt_re_process_res_rc_wc(wc
, cqe
);
2726 case CQ_BASE_CQE_TYPE_RES_UD
:
2727 if (qp
->qplib_qp
.id
==
2728 qp
->rdev
->qp1_sqp
->qplib_qp
.id
) {
2729 /* Handle this completion with
2730 * the stored completion
2735 bnxt_re_process_res_shadow_qp_wc
2740 bnxt_re_process_res_ud_wc(wc
, cqe
);
2743 dev_err(rdev_to_dev(cq
->rdev
),
2744 "POLL CQ : type 0x%x not handled",
2753 spin_unlock_irqrestore(&cq
->cq_lock
, flags
);
2754 return num_entries
- budget
;
2757 int bnxt_re_req_notify_cq(struct ib_cq
*ib_cq
,
2758 enum ib_cq_notify_flags ib_cqn_flags
)
2760 struct bnxt_re_cq
*cq
= container_of(ib_cq
, struct bnxt_re_cq
, ib_cq
);
2763 /* Trigger on the very next completion */
2764 if (ib_cqn_flags
& IB_CQ_NEXT_COMP
)
2765 type
= DBR_DBR_TYPE_CQ_ARMALL
;
2766 /* Trigger on the next solicited completion */
2767 else if (ib_cqn_flags
& IB_CQ_SOLICITED
)
2768 type
= DBR_DBR_TYPE_CQ_ARMSE
;
2770 bnxt_qplib_req_notify_cq(&cq
->qplib_cq
, type
);
2775 /* Memory Regions */
2776 struct ib_mr
*bnxt_re_get_dma_mr(struct ib_pd
*ib_pd
, int mr_access_flags
)
2778 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
2779 struct bnxt_re_dev
*rdev
= pd
->rdev
;
2780 struct bnxt_re_mr
*mr
;
2784 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
2786 return ERR_PTR(-ENOMEM
);
2789 mr
->qplib_mr
.pd
= &pd
->qplib_pd
;
2790 mr
->qplib_mr
.flags
= __from_ib_access_flags(mr_access_flags
);
2791 mr
->qplib_mr
.type
= CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR
;
2793 /* Allocate and register 0 as the address */
2794 rc
= bnxt_qplib_alloc_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
2798 mr
->qplib_mr
.hwq
.level
= PBL_LVL_MAX
;
2799 mr
->qplib_mr
.total_size
= -1; /* Infinte length */
2800 rc
= bnxt_qplib_reg_mr(&rdev
->qplib_res
, &mr
->qplib_mr
, &pbl
, 0, false);
2804 mr
->ib_mr
.lkey
= mr
->qplib_mr
.lkey
;
2805 if (mr_access_flags
& (IB_ACCESS_REMOTE_WRITE
| IB_ACCESS_REMOTE_READ
|
2806 IB_ACCESS_REMOTE_ATOMIC
))
2807 mr
->ib_mr
.rkey
= mr
->ib_mr
.lkey
;
2808 atomic_inc(&rdev
->mr_count
);
2813 bnxt_qplib_free_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
2819 int bnxt_re_dereg_mr(struct ib_mr
*ib_mr
)
2821 struct bnxt_re_mr
*mr
= container_of(ib_mr
, struct bnxt_re_mr
, ib_mr
);
2822 struct bnxt_re_dev
*rdev
= mr
->rdev
;
2825 if (mr
->npages
&& mr
->pages
) {
2826 rc
= bnxt_qplib_free_fast_reg_page_list(&rdev
->qplib_res
,
2832 rc
= bnxt_qplib_free_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
2834 if (!IS_ERR_OR_NULL(mr
->ib_umem
))
2835 ib_umem_release(mr
->ib_umem
);
2838 atomic_dec(&rdev
->mr_count
);
2842 static int bnxt_re_set_page(struct ib_mr
*ib_mr
, u64 addr
)
2844 struct bnxt_re_mr
*mr
= container_of(ib_mr
, struct bnxt_re_mr
, ib_mr
);
2846 if (unlikely(mr
->npages
== mr
->qplib_frpl
.max_pg_ptrs
))
2849 mr
->pages
[mr
->npages
++] = addr
;
2853 int bnxt_re_map_mr_sg(struct ib_mr
*ib_mr
, struct scatterlist
*sg
, int sg_nents
,
2854 unsigned int *sg_offset
)
2856 struct bnxt_re_mr
*mr
= container_of(ib_mr
, struct bnxt_re_mr
, ib_mr
);
2859 return ib_sg_to_pages(ib_mr
, sg
, sg_nents
, sg_offset
, bnxt_re_set_page
);
2862 struct ib_mr
*bnxt_re_alloc_mr(struct ib_pd
*ib_pd
, enum ib_mr_type type
,
2865 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
2866 struct bnxt_re_dev
*rdev
= pd
->rdev
;
2867 struct bnxt_re_mr
*mr
= NULL
;
2870 if (type
!= IB_MR_TYPE_MEM_REG
) {
2871 dev_dbg(rdev_to_dev(rdev
), "MR type 0x%x not supported", type
);
2872 return ERR_PTR(-EINVAL
);
2874 if (max_num_sg
> MAX_PBL_LVL_1_PGS
)
2875 return ERR_PTR(-EINVAL
);
2877 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
2879 return ERR_PTR(-ENOMEM
);
2882 mr
->qplib_mr
.pd
= &pd
->qplib_pd
;
2883 mr
->qplib_mr
.flags
= BNXT_QPLIB_FR_PMR
;
2884 mr
->qplib_mr
.type
= CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR
;
2886 rc
= bnxt_qplib_alloc_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
2890 mr
->ib_mr
.lkey
= mr
->qplib_mr
.lkey
;
2891 mr
->ib_mr
.rkey
= mr
->ib_mr
.lkey
;
2893 mr
->pages
= kcalloc(max_num_sg
, sizeof(u64
), GFP_KERNEL
);
2898 rc
= bnxt_qplib_alloc_fast_reg_page_list(&rdev
->qplib_res
,
2899 &mr
->qplib_frpl
, max_num_sg
);
2901 dev_err(rdev_to_dev(rdev
),
2902 "Failed to allocate HW FR page list");
2906 atomic_inc(&rdev
->mr_count
);
2910 bnxt_qplib_free_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
2917 /* Fast Memory Regions */
2918 struct ib_fmr
*bnxt_re_alloc_fmr(struct ib_pd
*ib_pd
, int mr_access_flags
,
2919 struct ib_fmr_attr
*fmr_attr
)
2921 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
2922 struct bnxt_re_dev
*rdev
= pd
->rdev
;
2923 struct bnxt_re_fmr
*fmr
;
2926 if (fmr_attr
->max_pages
> MAX_PBL_LVL_2_PGS
||
2927 fmr_attr
->max_maps
> rdev
->dev_attr
.max_map_per_fmr
) {
2928 dev_err(rdev_to_dev(rdev
), "Allocate FMR exceeded Max limit");
2929 return ERR_PTR(-ENOMEM
);
2931 fmr
= kzalloc(sizeof(*fmr
), GFP_KERNEL
);
2933 return ERR_PTR(-ENOMEM
);
2936 fmr
->qplib_fmr
.pd
= &pd
->qplib_pd
;
2937 fmr
->qplib_fmr
.type
= CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR
;
2939 rc
= bnxt_qplib_alloc_mrw(&rdev
->qplib_res
, &fmr
->qplib_fmr
);
2943 fmr
->qplib_fmr
.flags
= __from_ib_access_flags(mr_access_flags
);
2944 fmr
->ib_fmr
.lkey
= fmr
->qplib_fmr
.lkey
;
2945 fmr
->ib_fmr
.rkey
= fmr
->ib_fmr
.lkey
;
2947 atomic_inc(&rdev
->mr_count
);
2948 return &fmr
->ib_fmr
;
2954 int bnxt_re_map_phys_fmr(struct ib_fmr
*ib_fmr
, u64
*page_list
, int list_len
,
2957 struct bnxt_re_fmr
*fmr
= container_of(ib_fmr
, struct bnxt_re_fmr
,
2959 struct bnxt_re_dev
*rdev
= fmr
->rdev
;
2962 fmr
->qplib_fmr
.va
= iova
;
2963 fmr
->qplib_fmr
.total_size
= list_len
* PAGE_SIZE
;
2965 rc
= bnxt_qplib_reg_mr(&rdev
->qplib_res
, &fmr
->qplib_fmr
, page_list
,
2968 dev_err(rdev_to_dev(rdev
), "Failed to map FMR for lkey = 0x%x!",
2973 int bnxt_re_unmap_fmr(struct list_head
*fmr_list
)
2975 struct bnxt_re_dev
*rdev
;
2976 struct bnxt_re_fmr
*fmr
;
2977 struct ib_fmr
*ib_fmr
;
2980 /* Validate each FMRs inside the fmr_list */
2981 list_for_each_entry(ib_fmr
, fmr_list
, list
) {
2982 fmr
= container_of(ib_fmr
, struct bnxt_re_fmr
, ib_fmr
);
2986 rc
= bnxt_qplib_dereg_mrw(&rdev
->qplib_res
,
2987 &fmr
->qplib_fmr
, true);
2995 int bnxt_re_dealloc_fmr(struct ib_fmr
*ib_fmr
)
2997 struct bnxt_re_fmr
*fmr
= container_of(ib_fmr
, struct bnxt_re_fmr
,
2999 struct bnxt_re_dev
*rdev
= fmr
->rdev
;
3002 rc
= bnxt_qplib_free_mrw(&rdev
->qplib_res
, &fmr
->qplib_fmr
);
3004 dev_err(rdev_to_dev(rdev
), "Failed to free FMR");
3007 atomic_dec(&rdev
->mr_count
);
3012 struct ib_mr
*bnxt_re_reg_user_mr(struct ib_pd
*ib_pd
, u64 start
, u64 length
,
3013 u64 virt_addr
, int mr_access_flags
,
3014 struct ib_udata
*udata
)
3016 struct bnxt_re_pd
*pd
= container_of(ib_pd
, struct bnxt_re_pd
, ib_pd
);
3017 struct bnxt_re_dev
*rdev
= pd
->rdev
;
3018 struct bnxt_re_mr
*mr
;
3019 struct ib_umem
*umem
;
3020 u64
*pbl_tbl
, *pbl_tbl_orig
;
3021 int i
, umem_pgs
, pages
, rc
;
3022 struct scatterlist
*sg
;
3025 mr
= kzalloc(sizeof(*mr
), GFP_KERNEL
);
3027 return ERR_PTR(-ENOMEM
);
3030 mr
->qplib_mr
.pd
= &pd
->qplib_pd
;
3031 mr
->qplib_mr
.flags
= __from_ib_access_flags(mr_access_flags
);
3032 mr
->qplib_mr
.type
= CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR
;
3034 umem
= ib_umem_get(ib_pd
->uobject
->context
, start
, length
,
3035 mr_access_flags
, 0);
3037 dev_err(rdev_to_dev(rdev
), "Failed to get umem");
3043 rc
= bnxt_qplib_alloc_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
3045 dev_err(rdev_to_dev(rdev
), "Failed to allocate MR");
3048 /* The fixed portion of the rkey is the same as the lkey */
3049 mr
->ib_mr
.rkey
= mr
->qplib_mr
.rkey
;
3051 mr
->qplib_mr
.va
= virt_addr
;
3052 umem_pgs
= ib_umem_page_count(umem
);
3054 dev_err(rdev_to_dev(rdev
), "umem is invalid!");
3058 mr
->qplib_mr
.total_size
= length
;
3060 pbl_tbl
= kcalloc(umem_pgs
, sizeof(u64
*), GFP_KERNEL
);
3065 pbl_tbl_orig
= pbl_tbl
;
3067 if (umem
->hugetlb
) {
3068 dev_err(rdev_to_dev(rdev
), "umem hugetlb not supported!");
3073 if (umem
->page_shift
!= PAGE_SHIFT
) {
3074 dev_err(rdev_to_dev(rdev
), "umem page shift unsupported!");
3078 /* Map umem buf ptrs to the PBL */
3079 for_each_sg(umem
->sg_head
.sgl
, sg
, umem
->nmap
, entry
) {
3080 pages
= sg_dma_len(sg
) >> umem
->page_shift
;
3081 for (i
= 0; i
< pages
; i
++, pbl_tbl
++)
3082 *pbl_tbl
= sg_dma_address(sg
) + (i
<< umem
->page_shift
);
3084 rc
= bnxt_qplib_reg_mr(&rdev
->qplib_res
, &mr
->qplib_mr
, pbl_tbl_orig
,
3087 dev_err(rdev_to_dev(rdev
), "Failed to register user MR");
3091 kfree(pbl_tbl_orig
);
3093 mr
->ib_mr
.lkey
= mr
->qplib_mr
.lkey
;
3094 mr
->ib_mr
.rkey
= mr
->qplib_mr
.lkey
;
3095 atomic_inc(&rdev
->mr_count
);
3099 kfree(pbl_tbl_orig
);
3101 bnxt_qplib_free_mrw(&rdev
->qplib_res
, &mr
->qplib_mr
);
3103 ib_umem_release(umem
);
3109 struct ib_ucontext
*bnxt_re_alloc_ucontext(struct ib_device
*ibdev
,
3110 struct ib_udata
*udata
)
3112 struct bnxt_re_dev
*rdev
= to_bnxt_re_dev(ibdev
, ibdev
);
3113 struct bnxt_re_uctx_resp resp
;
3114 struct bnxt_re_ucontext
*uctx
;
3115 struct bnxt_qplib_dev_attr
*dev_attr
= &rdev
->dev_attr
;
3118 dev_dbg(rdev_to_dev(rdev
), "ABI version requested %d",
3119 ibdev
->uverbs_abi_ver
);
3121 if (ibdev
->uverbs_abi_ver
!= BNXT_RE_ABI_VERSION
) {
3122 dev_dbg(rdev_to_dev(rdev
), " is different from the device %d ",
3123 BNXT_RE_ABI_VERSION
);
3124 return ERR_PTR(-EPERM
);
3127 uctx
= kzalloc(sizeof(*uctx
), GFP_KERNEL
);
3129 return ERR_PTR(-ENOMEM
);
3133 uctx
->shpg
= (void *)__get_free_page(GFP_KERNEL
);
3138 spin_lock_init(&uctx
->sh_lock
);
3140 resp
.dev_id
= rdev
->en_dev
->pdev
->devfn
; /*Temp, Use idr_alloc instead*/
3141 resp
.max_qp
= rdev
->qplib_ctx
.qpc_count
;
3142 resp
.pg_size
= PAGE_SIZE
;
3143 resp
.cqe_sz
= sizeof(struct cq_base
);
3144 resp
.max_cqd
= dev_attr
->max_cq_wqes
;
3147 rc
= ib_copy_to_udata(udata
, &resp
, sizeof(resp
));
3149 dev_err(rdev_to_dev(rdev
), "Failed to copy user context");
3154 return &uctx
->ib_uctx
;
3156 free_page((unsigned long)uctx
->shpg
);
3163 int bnxt_re_dealloc_ucontext(struct ib_ucontext
*ib_uctx
)
3165 struct bnxt_re_ucontext
*uctx
= container_of(ib_uctx
,
3166 struct bnxt_re_ucontext
,
3169 free_page((unsigned long)uctx
->shpg
);
3174 /* Helper function to mmap the virtual memory from user app */
3175 int bnxt_re_mmap(struct ib_ucontext
*ib_uctx
, struct vm_area_struct
*vma
)
3177 struct bnxt_re_ucontext
*uctx
= container_of(ib_uctx
,
3178 struct bnxt_re_ucontext
,
3180 struct bnxt_re_dev
*rdev
= uctx
->rdev
;
3183 if (vma
->vm_end
- vma
->vm_start
!= PAGE_SIZE
)
3186 if (vma
->vm_pgoff
) {
3187 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
3188 if (io_remap_pfn_range(vma
, vma
->vm_start
, vma
->vm_pgoff
,
3189 PAGE_SIZE
, vma
->vm_page_prot
)) {
3190 dev_err(rdev_to_dev(rdev
), "Failed to map DPI");
3194 pfn
= virt_to_phys(uctx
->shpg
) >> PAGE_SHIFT
;
3195 if (remap_pfn_range(vma
, vma
->vm_start
,
3196 pfn
, PAGE_SIZE
, vma
->vm_page_prot
)) {
3197 dev_err(rdev_to_dev(rdev
),
3198 "Failed to map shared page");