projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
net/mlx5: Fix mlx5_get_vector_affinity function
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / infiniband / hw / mlx5 / main.c
1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/errno.h>
38 #include <linux/pci.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/slab.h>
41 #if defined(CONFIG_X86)
42 #include <asm/pat.h>
43 #endif
44 #include <linux/sched.h>
45 #include <linux/sched/mm.h>
46 #include <linux/sched/task.h>
47 #include <linux/delay.h>
48 #include <rdma/ib_user_verbs.h>
49 #include <rdma/ib_addr.h>
50 #include <rdma/ib_cache.h>
51 #include <linux/mlx5/port.h>
52 #include <linux/mlx5/vport.h>
53 #include <linux/list.h>
54 #include <rdma/ib_smi.h>
55 #include <rdma/ib_umem.h>
56 #include <linux/in.h>
57 #include <linux/etherdevice.h>
58 #include <linux/mlx5/fs.h>
59 #include <linux/mlx5/vport.h>
60 #include "mlx5_ib.h"
61 #include "cmd.h"
62 #include <linux/mlx5/vport.h>
63
64 #define DRIVER_NAME "mlx5_ib"
65 #define DRIVER_VERSION "5.0-0"
66
67 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
68 MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
69 MODULE_LICENSE("Dual BSD/GPL");
70
71 static char mlx5_version[] =
72 DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
73 DRIVER_VERSION "\n";
74
75 enum {
76 MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
77 };
78
79 static enum rdma_link_layer
80 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
81 {
82 switch (port_type_cap) {
83 case MLX5_CAP_PORT_TYPE_IB:
84 return IB_LINK_LAYER_INFINIBAND;
85 case MLX5_CAP_PORT_TYPE_ETH:
86 return IB_LINK_LAYER_ETHERNET;
87 default:
88 return IB_LINK_LAYER_UNSPECIFIED;
89 }
90 }
91
92 static enum rdma_link_layer
93 mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
94 {
95 struct mlx5_ib_dev *dev = to_mdev(device);
96 int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
97
98 return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
99 }
100
101 static int get_port_state(struct ib_device *ibdev,
102 u8 port_num,
103 enum ib_port_state *state)
104 {
105 struct ib_port_attr attr;
106 int ret;
107
108 memset(&attr, 0, sizeof(attr));
109 ret = mlx5_ib_query_port(ibdev, port_num, &attr);
110 if (!ret)
111 *state = attr.state;
112 return ret;
113 }
114
115 static int mlx5_netdev_event(struct notifier_block *this,
116 unsigned long event, void *ptr)
117 {
118 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
119 struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
120 roce.nb);
121
122 switch (event) {
123 case NETDEV_REGISTER:
124 case NETDEV_UNREGISTER:
125 write_lock(&ibdev->roce.netdev_lock);
126 if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
127 ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ?
128 NULL : ndev;
129 write_unlock(&ibdev->roce.netdev_lock);
130 break;
131
132 case NETDEV_CHANGE:
133 case NETDEV_UP:
134 case NETDEV_DOWN: {
135 struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
136 struct net_device *upper = NULL;
137
138 if (lag_ndev) {
139 upper = netdev_master_upper_dev_get(lag_ndev);
140 dev_put(lag_ndev);
141 }
142
143 if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
144 && ibdev->ib_active) {
145 struct ib_event ibev = { };
146 enum ib_port_state port_state;
147
148 if (get_port_state(&ibdev->ib_dev, 1, &port_state))
149 return NOTIFY_DONE;
150
151 if (ibdev->roce.last_port_state == port_state)
152 return NOTIFY_DONE;
153
154 ibdev->roce.last_port_state = port_state;
155 ibev.device = &ibdev->ib_dev;
156 if (port_state == IB_PORT_DOWN)
157 ibev.event = IB_EVENT_PORT_ERR;
158 else if (port_state == IB_PORT_ACTIVE)
159 ibev.event = IB_EVENT_PORT_ACTIVE;
160 else
161 return NOTIFY_DONE;
162
163 ibev.element.port_num = 1;
164 ib_dispatch_event(&ibev);
165 }
166 break;
167 }
168
169 default:
170 break;
171 }
172
173 return NOTIFY_DONE;
174 }
175
176 static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
177 u8 port_num)
178 {
179 struct mlx5_ib_dev *ibdev = to_mdev(device);
180 struct net_device *ndev;
181
182 ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
183 if (ndev)
184 return ndev;
185
186 /* Ensure ndev does not disappear before we invoke dev_hold()
187 */
188 read_lock(&ibdev->roce.netdev_lock);
189 ndev = ibdev->roce.netdev;
190 if (ndev)
191 dev_hold(ndev);
192 read_unlock(&ibdev->roce.netdev_lock);
193
194 return ndev;
195 }
196
197 static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
198 u8 *active_width)
199 {
200 switch (eth_proto_oper) {
201 case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
202 case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
203 case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
204 case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
205 *active_width = IB_WIDTH_1X;
206 *active_speed = IB_SPEED_SDR;
207 break;
208 case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
209 case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
210 case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
211 case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
212 case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
213 case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
214 case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
215 *active_width = IB_WIDTH_1X;
216 *active_speed = IB_SPEED_QDR;
217 break;
218 case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
219 case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
220 case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
221 *active_width = IB_WIDTH_1X;
222 *active_speed = IB_SPEED_EDR;
223 break;
224 case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
225 case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
226 case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
227 case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
228 *active_width = IB_WIDTH_4X;
229 *active_speed = IB_SPEED_QDR;
230 break;
231 case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
232 case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
233 case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
234 *active_width = IB_WIDTH_1X;
235 *active_speed = IB_SPEED_HDR;
236 break;
237 case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
238 *active_width = IB_WIDTH_4X;
239 *active_speed = IB_SPEED_FDR;
240 break;
241 case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
242 case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
243 case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
244 case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
245 *active_width = IB_WIDTH_4X;
246 *active_speed = IB_SPEED_EDR;
247 break;
248 default:
249 return -EINVAL;
250 }
251
252 return 0;
253 }
254
255 static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
256 struct ib_port_attr *props)
257 {
258 struct mlx5_ib_dev *dev = to_mdev(device);
259 struct mlx5_core_dev *mdev = dev->mdev;
260 struct net_device *ndev, *upper;
261 enum ib_mtu ndev_ib_mtu;
262 u16 qkey_viol_cntr;
263 u32 eth_prot_oper;
264 int err;
265
266 /* Possible bad flows are checked before filling out props so in case
267 * of an error it will still be zeroed out.
268 */
269 err = mlx5_query_port_eth_proto_oper(mdev, &eth_prot_oper, port_num);
270 if (err)
271 return err;
272
273 props->active_width = IB_WIDTH_4X;
274 props->active_speed = IB_SPEED_QDR;
275
276 translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
277 &props->active_width);
278
279 props->port_cap_flags |= IB_PORT_CM_SUP;
280 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
281
282 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
283 roce_address_table_size);
284 props->max_mtu = IB_MTU_4096;
285 props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
286 props->pkey_tbl_len = 1;
287 props->state = IB_PORT_DOWN;
288 props->phys_state = 3;
289
290 mlx5_query_nic_vport_qkey_viol_cntr(dev->mdev, &qkey_viol_cntr);
291 props->qkey_viol_cntr = qkey_viol_cntr;
292
293 ndev = mlx5_ib_get_netdev(device, port_num);
294 if (!ndev)
295 return 0;
296
297 if (mlx5_lag_is_active(dev->mdev)) {
298 rcu_read_lock();
299 upper = netdev_master_upper_dev_get_rcu(ndev);
300 if (upper) {
301 dev_put(ndev);
302 ndev = upper;
303 dev_hold(ndev);
304 }
305 rcu_read_unlock();
306 }
307
308 if (netif_running(ndev) && netif_carrier_ok(ndev)) {
309 props->state = IB_PORT_ACTIVE;
310 props->phys_state = 5;
311 }
312
313 ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
314
315 dev_put(ndev);
316
317 props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
318 return 0;
319 }
320
321 static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
322 unsigned int index, const union ib_gid *gid,
323 const struct ib_gid_attr *attr)
324 {
325 enum ib_gid_type gid_type = IB_GID_TYPE_IB;
326 u8 roce_version = 0;
327 u8 roce_l3_type = 0;
328 bool vlan = false;
329 u8 mac[ETH_ALEN];
330 u16 vlan_id = 0;
331
332 if (gid) {
333 gid_type = attr->gid_type;
334 ether_addr_copy(mac, attr->ndev->dev_addr);
335
336 if (is_vlan_dev(attr->ndev)) {
337 vlan = true;
338 vlan_id = vlan_dev_vlan_id(attr->ndev);
339 }
340 }
341
342 switch (gid_type) {
343 case IB_GID_TYPE_IB:
344 roce_version = MLX5_ROCE_VERSION_1;
345 break;
346 case IB_GID_TYPE_ROCE_UDP_ENCAP:
347 roce_version = MLX5_ROCE_VERSION_2;
348 if (ipv6_addr_v4mapped((void *)gid))
349 roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
350 else
351 roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
352 break;
353
354 default:
355 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
356 }
357
358 return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
359 roce_l3_type, gid->raw, mac, vlan,
360 vlan_id);
361 }
362
363 static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
364 unsigned int index, const union ib_gid *gid,
365 const struct ib_gid_attr *attr,
366 __always_unused void **context)
367 {
368 return set_roce_addr(to_mdev(device), port_num, index, gid, attr);
369 }
370
371 static int mlx5_ib_del_gid(struct ib_device *device, u8 port_num,
372 unsigned int index, __always_unused void **context)
373 {
374 return set_roce_addr(to_mdev(device), port_num, index, NULL, NULL);
375 }
376
377 __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
378 int index)
379 {
380 struct ib_gid_attr attr;
381 union ib_gid gid;
382
383 if (ib_get_cached_gid(&dev->ib_dev, port_num, index, &gid, &attr))
384 return 0;
385
386 if (!attr.ndev)
387 return 0;
388
389 dev_put(attr.ndev);
390
391 if (attr.gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
392 return 0;
393
394 return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
395 }
396
397 int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
398 int index, enum ib_gid_type *gid_type)
399 {
400 struct ib_gid_attr attr;
401 union ib_gid gid;
402 int ret;
403
404 ret = ib_get_cached_gid(&dev->ib_dev, port_num, index, &gid, &attr);
405 if (ret)
406 return ret;
407
408 if (!attr.ndev)
409 return -ENODEV;
410
411 dev_put(attr.ndev);
412
413 *gid_type = attr.gid_type;
414
415 return 0;
416 }
417
418 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
419 {
420 if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
421 return !MLX5_CAP_GEN(dev->mdev, ib_virt);
422 return 0;
423 }
424
425 enum {
426 MLX5_VPORT_ACCESS_METHOD_MAD,
427 MLX5_VPORT_ACCESS_METHOD_HCA,
428 MLX5_VPORT_ACCESS_METHOD_NIC,
429 };
430
431 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
432 {
433 if (mlx5_use_mad_ifc(to_mdev(ibdev)))
434 return MLX5_VPORT_ACCESS_METHOD_MAD;
435
436 if (mlx5_ib_port_link_layer(ibdev, 1) ==
437 IB_LINK_LAYER_ETHERNET)
438 return MLX5_VPORT_ACCESS_METHOD_NIC;
439
440 return MLX5_VPORT_ACCESS_METHOD_HCA;
441 }
442
443 static void get_atomic_caps(struct mlx5_ib_dev *dev,
444 struct ib_device_attr *props)
445 {
446 u8 tmp;
447 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
448 u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
449 u8 atomic_req_8B_endianness_mode =
450 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
451
452 /* Check if HW supports 8 bytes standard atomic operations and capable
453 * of host endianness respond
454 */
455 tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
456 if (((atomic_operations & tmp) == tmp) &&
457 (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
458 (atomic_req_8B_endianness_mode)) {
459 props->atomic_cap = IB_ATOMIC_HCA;
460 } else {
461 props->atomic_cap = IB_ATOMIC_NONE;
462 }
463 }
464
465 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
466 __be64 *sys_image_guid)
467 {
468 struct mlx5_ib_dev *dev = to_mdev(ibdev);
469 struct mlx5_core_dev *mdev = dev->mdev;
470 u64 tmp;
471 int err;
472
473 switch (mlx5_get_vport_access_method(ibdev)) {
474 case MLX5_VPORT_ACCESS_METHOD_MAD:
475 return mlx5_query_mad_ifc_system_image_guid(ibdev,
476 sys_image_guid);
477
478 case MLX5_VPORT_ACCESS_METHOD_HCA:
479 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
480 break;
481
482 case MLX5_VPORT_ACCESS_METHOD_NIC:
483 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
484 break;
485
486 default:
487 return -EINVAL;
488 }
489
490 if (!err)
491 *sys_image_guid = cpu_to_be64(tmp);
492
493 return err;
494
495 }
496
497 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
498 u16 *max_pkeys)
499 {
500 struct mlx5_ib_dev *dev = to_mdev(ibdev);
501 struct mlx5_core_dev *mdev = dev->mdev;
502
503 switch (mlx5_get_vport_access_method(ibdev)) {
504 case MLX5_VPORT_ACCESS_METHOD_MAD:
505 return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
506
507 case MLX5_VPORT_ACCESS_METHOD_HCA:
508 case MLX5_VPORT_ACCESS_METHOD_NIC:
509 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
510 pkey_table_size));
511 return 0;
512
513 default:
514 return -EINVAL;
515 }
516 }
517
518 static int mlx5_query_vendor_id(struct ib_device *ibdev,
519 u32 *vendor_id)
520 {
521 struct mlx5_ib_dev *dev = to_mdev(ibdev);
522
523 switch (mlx5_get_vport_access_method(ibdev)) {
524 case MLX5_VPORT_ACCESS_METHOD_MAD:
525 return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
526
527 case MLX5_VPORT_ACCESS_METHOD_HCA:
528 case MLX5_VPORT_ACCESS_METHOD_NIC:
529 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
530
531 default:
532 return -EINVAL;
533 }
534 }
535
536 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
537 __be64 *node_guid)
538 {
539 u64 tmp;
540 int err;
541
542 switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
543 case MLX5_VPORT_ACCESS_METHOD_MAD:
544 return mlx5_query_mad_ifc_node_guid(dev, node_guid);
545
546 case MLX5_VPORT_ACCESS_METHOD_HCA:
547 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
548 break;
549
550 case MLX5_VPORT_ACCESS_METHOD_NIC:
551 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
552 break;
553
554 default:
555 return -EINVAL;
556 }
557
558 if (!err)
559 *node_guid = cpu_to_be64(tmp);
560
561 return err;
562 }
563
564 struct mlx5_reg_node_desc {
565 u8 desc[IB_DEVICE_NODE_DESC_MAX];
566 };
567
568 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
569 {
570 struct mlx5_reg_node_desc in;
571
572 if (mlx5_use_mad_ifc(dev))
573 return mlx5_query_mad_ifc_node_desc(dev, node_desc);
574
575 memset(&in, 0, sizeof(in));
576
577 return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
578 sizeof(struct mlx5_reg_node_desc),
579 MLX5_REG_NODE_DESC, 0, 0);
580 }
581
582 static int mlx5_ib_query_device(struct ib_device *ibdev,
583 struct ib_device_attr *props,
584 struct ib_udata *uhw)
585 {
586 struct mlx5_ib_dev *dev = to_mdev(ibdev);
587 struct mlx5_core_dev *mdev = dev->mdev;
588 int err = -ENOMEM;
589 int max_sq_desc;
590 int max_rq_sg;
591 int max_sq_sg;
592 u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
593 struct mlx5_ib_query_device_resp resp = {};
594 size_t resp_len;
595 u64 max_tso;
596
597 resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
598 if (uhw->outlen && uhw->outlen < resp_len)
599 return -EINVAL;
600 else
601 resp.response_length = resp_len;
602
603 if (uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
604 return -EINVAL;
605
606 memset(props, 0, sizeof(*props));
607 err = mlx5_query_system_image_guid(ibdev,
608 &props->sys_image_guid);
609 if (err)
610 return err;
611
612 err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
613 if (err)
614 return err;
615
616 err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
617 if (err)
618 return err;
619
620 props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
621 (fw_rev_min(dev->mdev) << 16) |
622 fw_rev_sub(dev->mdev);
623 props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
624 IB_DEVICE_PORT_ACTIVE_EVENT |
625 IB_DEVICE_SYS_IMAGE_GUID |
626 IB_DEVICE_RC_RNR_NAK_GEN;
627
628 if (MLX5_CAP_GEN(mdev, pkv))
629 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
630 if (MLX5_CAP_GEN(mdev, qkv))
631 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
632 if (MLX5_CAP_GEN(mdev, apm))
633 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
634 if (MLX5_CAP_GEN(mdev, xrc))
635 props->device_cap_flags |= IB_DEVICE_XRC;
636 if (MLX5_CAP_GEN(mdev, imaicl)) {
637 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
638 IB_DEVICE_MEM_WINDOW_TYPE_2B;
639 props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
640 /* We support 'Gappy' memory registration too */
641 props->device_cap_flags |= IB_DEVICE_SG_GAPS_REG;
642 }
643 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
644 if (MLX5_CAP_GEN(mdev, sho)) {
645 props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
646 /* At this stage no support for signature handover */
647 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
648 IB_PROT_T10DIF_TYPE_2 |
649 IB_PROT_T10DIF_TYPE_3;
650 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
651 IB_GUARD_T10DIF_CSUM;
652 }
653 if (MLX5_CAP_GEN(mdev, block_lb_mc))
654 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
655
656 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads)) {
657 if (MLX5_CAP_ETH(mdev, csum_cap)) {
658 /* Legacy bit to support old userspace libraries */
659 props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
660 props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
661 }
662
663 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
664 props->raw_packet_caps |=
665 IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
666
667 if (field_avail(typeof(resp), tso_caps, uhw->outlen)) {
668 max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
669 if (max_tso) {
670 resp.tso_caps.max_tso = 1 << max_tso;
671 resp.tso_caps.supported_qpts |=
672 1 << IB_QPT_RAW_PACKET;
673 resp.response_length += sizeof(resp.tso_caps);
674 }
675 }
676
677 if (field_avail(typeof(resp), rss_caps, uhw->outlen)) {
678 resp.rss_caps.rx_hash_function =
679 MLX5_RX_HASH_FUNC_TOEPLITZ;
680 resp.rss_caps.rx_hash_fields_mask =
681 MLX5_RX_HASH_SRC_IPV4 |
682 MLX5_RX_HASH_DST_IPV4 |
683 MLX5_RX_HASH_SRC_IPV6 |
684 MLX5_RX_HASH_DST_IPV6 |
685 MLX5_RX_HASH_SRC_PORT_TCP |
686 MLX5_RX_HASH_DST_PORT_TCP |
687 MLX5_RX_HASH_SRC_PORT_UDP |
688 MLX5_RX_HASH_DST_PORT_UDP;
689 resp.response_length += sizeof(resp.rss_caps);
690 }
691 } else {
692 if (field_avail(typeof(resp), tso_caps, uhw->outlen))
693 resp.response_length += sizeof(resp.tso_caps);
694 if (field_avail(typeof(resp), rss_caps, uhw->outlen))
695 resp.response_length += sizeof(resp.rss_caps);
696 }
697
698 if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
699 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
700 props->device_cap_flags |= IB_DEVICE_UD_TSO;
701 }
702
703 if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
704 MLX5_CAP_GEN(dev->mdev, general_notification_event))
705 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
706
707 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
708 MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
709 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
710
711 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
712 MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
713 /* Legacy bit to support old userspace libraries */
714 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
715 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
716 }
717
718 if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
719 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
720
721 props->vendor_part_id = mdev->pdev->device;
722 props->hw_ver = mdev->pdev->revision;
723
724 props->max_mr_size = ~0ull;
725 props->page_size_cap = ~(min_page_size - 1);
726 props->max_qp = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
727 props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
728 max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
729 sizeof(struct mlx5_wqe_data_seg);
730 max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
731 max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
732 sizeof(struct mlx5_wqe_raddr_seg)) /
733 sizeof(struct mlx5_wqe_data_seg);
734 props->max_sge = min(max_rq_sg, max_sq_sg);
735 props->max_sge_rd = MLX5_MAX_SGE_RD;
736 props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
737 props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
738 props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
739 props->max_pd = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
740 props->max_qp_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
741 props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
742 props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
743 props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
744 props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
745 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
746 props->max_srq_sge = max_rq_sg - 1;
747 props->max_fast_reg_page_list_len =
748 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
749 get_atomic_caps(dev, props);
750 props->masked_atomic_cap = IB_ATOMIC_NONE;
751 props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
752 props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
753 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
754 props->max_mcast_grp;
755 props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
756 props->max_ah = INT_MAX;
757 props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
758 props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
759
760 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
761 if (MLX5_CAP_GEN(mdev, pg))
762 props->device_cap_flags |= IB_DEVICE_ON_DEMAND_PAGING;
763 props->odp_caps = dev->odp_caps;
764 #endif
765
766 if (MLX5_CAP_GEN(mdev, cd))
767 props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
768
769 if (!mlx5_core_is_pf(mdev))
770 props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
771
772 if (mlx5_ib_port_link_layer(ibdev, 1) ==
773 IB_LINK_LAYER_ETHERNET) {
774 props->rss_caps.max_rwq_indirection_tables =
775 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
776 props->rss_caps.max_rwq_indirection_table_size =
777 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
778 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
779 props->max_wq_type_rq =
780 1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
781 }
782
783 if (MLX5_CAP_GEN(mdev, tag_matching)) {
784 props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
785 props->tm_caps.max_num_tags =
786 (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
787 props->tm_caps.flags = IB_TM_CAP_RC;
788 props->tm_caps.max_ops =
789 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
790 props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
791 }
792
793 if (field_avail(typeof(resp), cqe_comp_caps, uhw->outlen)) {
794 resp.cqe_comp_caps.max_num =
795 MLX5_CAP_GEN(dev->mdev, cqe_compression) ?
796 MLX5_CAP_GEN(dev->mdev, cqe_compression_max_num) : 0;
797 resp.cqe_comp_caps.supported_format =
798 MLX5_IB_CQE_RES_FORMAT_HASH |
799 MLX5_IB_CQE_RES_FORMAT_CSUM;
800 resp.response_length += sizeof(resp.cqe_comp_caps);
801 }
802
803 if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen)) {
804 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
805 MLX5_CAP_GEN(mdev, qos)) {
806 resp.packet_pacing_caps.qp_rate_limit_max =
807 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
808 resp.packet_pacing_caps.qp_rate_limit_min =
809 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
810 resp.packet_pacing_caps.supported_qpts |=
811 1 << IB_QPT_RAW_PACKET;
812 }
813 resp.response_length += sizeof(resp.packet_pacing_caps);
814 }
815
816 if (field_avail(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes,
817 uhw->outlen)) {
818 if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
819 resp.mlx5_ib_support_multi_pkt_send_wqes =
820 MLX5_IB_ALLOW_MPW;
821
822 if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
823 resp.mlx5_ib_support_multi_pkt_send_wqes |=
824 MLX5_IB_SUPPORT_EMPW;
825
826 resp.response_length +=
827 sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
828 }
829
830 if (field_avail(typeof(resp), reserved, uhw->outlen))
831 resp.response_length += sizeof(resp.reserved);
832
833 if (field_avail(typeof(resp), sw_parsing_caps,
834 uhw->outlen)) {
835 resp.response_length += sizeof(resp.sw_parsing_caps);
836 if (MLX5_CAP_ETH(mdev, swp)) {
837 resp.sw_parsing_caps.sw_parsing_offloads |=
838 MLX5_IB_SW_PARSING;
839
840 if (MLX5_CAP_ETH(mdev, swp_csum))
841 resp.sw_parsing_caps.sw_parsing_offloads |=
842 MLX5_IB_SW_PARSING_CSUM;
843
844 if (MLX5_CAP_ETH(mdev, swp_lso))
845 resp.sw_parsing_caps.sw_parsing_offloads |=
846 MLX5_IB_SW_PARSING_LSO;
847
848 if (resp.sw_parsing_caps.sw_parsing_offloads)
849 resp.sw_parsing_caps.supported_qpts =
850 BIT(IB_QPT_RAW_PACKET);
851 }
852 }
853
854 if (uhw->outlen) {
855 err = ib_copy_to_udata(uhw, &resp, resp.response_length);
856
857 if (err)
858 return err;
859 }
860
861 return 0;
862 }
863
864 enum mlx5_ib_width {
865 MLX5_IB_WIDTH_1X = 1 << 0,
866 MLX5_IB_WIDTH_2X = 1 << 1,
867 MLX5_IB_WIDTH_4X = 1 << 2,
868 MLX5_IB_WIDTH_8X = 1 << 3,
869 MLX5_IB_WIDTH_12X = 1 << 4
870 };
871
872 static int translate_active_width(struct ib_device *ibdev, u8 active_width,
873 u8 *ib_width)
874 {
875 struct mlx5_ib_dev *dev = to_mdev(ibdev);
876 int err = 0;
877
878 if (active_width & MLX5_IB_WIDTH_1X) {
879 *ib_width = IB_WIDTH_1X;
880 } else if (active_width & MLX5_IB_WIDTH_2X) {
881 mlx5_ib_dbg(dev, "active_width %d is not supported by IB spec\n",
882 (int)active_width);
883 err = -EINVAL;
884 } else if (active_width & MLX5_IB_WIDTH_4X) {
885 *ib_width = IB_WIDTH_4X;
886 } else if (active_width & MLX5_IB_WIDTH_8X) {
887 *ib_width = IB_WIDTH_8X;
888 } else if (active_width & MLX5_IB_WIDTH_12X) {
889 *ib_width = IB_WIDTH_12X;
890 } else {
891 mlx5_ib_dbg(dev, "Invalid active_width %d\n",
892 (int)active_width);
893 err = -EINVAL;
894 }
895
896 return err;
897 }
898
899 static int mlx5_mtu_to_ib_mtu(int mtu)
900 {
901 switch (mtu) {
902 case 256: return 1;
903 case 512: return 2;
904 case 1024: return 3;
905 case 2048: return 4;
906 case 4096: return 5;
907 default:
908 pr_warn("invalid mtu\n");
909 return -1;
910 }
911 }
912
913 enum ib_max_vl_num {
914 __IB_MAX_VL_0 = 1,
915 __IB_MAX_VL_0_1 = 2,
916 __IB_MAX_VL_0_3 = 3,
917 __IB_MAX_VL_0_7 = 4,
918 __IB_MAX_VL_0_14 = 5,
919 };
920
921 enum mlx5_vl_hw_cap {
922 MLX5_VL_HW_0 = 1,
923 MLX5_VL_HW_0_1 = 2,
924 MLX5_VL_HW_0_2 = 3,
925 MLX5_VL_HW_0_3 = 4,
926 MLX5_VL_HW_0_4 = 5,
927 MLX5_VL_HW_0_5 = 6,
928 MLX5_VL_HW_0_6 = 7,
929 MLX5_VL_HW_0_7 = 8,
930 MLX5_VL_HW_0_14 = 15
931 };
932
933 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
934 u8 *max_vl_num)
935 {
936 switch (vl_hw_cap) {
937 case MLX5_VL_HW_0:
938 *max_vl_num = __IB_MAX_VL_0;
939 break;
940 case MLX5_VL_HW_0_1:
941 *max_vl_num = __IB_MAX_VL_0_1;
942 break;
943 case MLX5_VL_HW_0_3:
944 *max_vl_num = __IB_MAX_VL_0_3;
945 break;
946 case MLX5_VL_HW_0_7:
947 *max_vl_num = __IB_MAX_VL_0_7;
948 break;
949 case MLX5_VL_HW_0_14:
950 *max_vl_num = __IB_MAX_VL_0_14;
951 break;
952
953 default:
954 return -EINVAL;
955 }
956
957 return 0;
958 }
959
960 static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
961 struct ib_port_attr *props)
962 {
963 struct mlx5_ib_dev *dev = to_mdev(ibdev);
964 struct mlx5_core_dev *mdev = dev->mdev;
965 struct mlx5_hca_vport_context *rep;
966 u16 max_mtu;
967 u16 oper_mtu;
968 int err;
969 u8 ib_link_width_oper;
970 u8 vl_hw_cap;
971
972 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
973 if (!rep) {
974 err = -ENOMEM;
975 goto out;
976 }
977
978 /* props being zeroed by the caller, avoid zeroing it here */
979
980 err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
981 if (err)
982 goto out;
983
984 props->lid = rep->lid;
985 props->lmc = rep->lmc;
986 props->sm_lid = rep->sm_lid;
987 props->sm_sl = rep->sm_sl;
988 props->state = rep->vport_state;
989 props->phys_state = rep->port_physical_state;
990 props->port_cap_flags = rep->cap_mask1;
991 props->gid_tbl_len = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
992 props->max_msg_sz = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
993 props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
994 props->bad_pkey_cntr = rep->pkey_violation_counter;
995 props->qkey_viol_cntr = rep->qkey_violation_counter;
996 props->subnet_timeout = rep->subnet_timeout;
997 props->init_type_reply = rep->init_type_reply;
998 props->grh_required = rep->grh_required;
999
1000 err = mlx5_query_port_link_width_oper(mdev, &ib_link_width_oper, port);
1001 if (err)
1002 goto out;
1003
1004 err = translate_active_width(ibdev, ib_link_width_oper,
1005 &props->active_width);
1006 if (err)
1007 goto out;
1008 err = mlx5_query_port_ib_proto_oper(mdev, &props->active_speed, port);
1009 if (err)
1010 goto out;
1011
1012 mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1013
1014 props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1015
1016 mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1017
1018 props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1019
1020 err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1021 if (err)
1022 goto out;
1023
1024 err = translate_max_vl_num(ibdev, vl_hw_cap,
1025 &props->max_vl_num);
1026 out:
1027 kfree(rep);
1028 return err;
1029 }
1030
1031 int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
1032 struct ib_port_attr *props)
1033 {
1034 unsigned int count;
1035 int ret;
1036
1037 switch (mlx5_get_vport_access_method(ibdev)) {
1038 case MLX5_VPORT_ACCESS_METHOD_MAD:
1039 ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1040 break;
1041
1042 case MLX5_VPORT_ACCESS_METHOD_HCA:
1043 ret = mlx5_query_hca_port(ibdev, port, props);
1044 break;
1045
1046 case MLX5_VPORT_ACCESS_METHOD_NIC:
1047 ret = mlx5_query_port_roce(ibdev, port, props);
1048 break;
1049
1050 default:
1051 ret = -EINVAL;
1052 }
1053
1054 if (!ret && props) {
1055 count = mlx5_core_reserved_gids_count(to_mdev(ibdev)->mdev);
1056 props->gid_tbl_len -= count;
1057 }
1058 return ret;
1059 }
1060
1061 static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
1062 union ib_gid *gid)
1063 {
1064 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1065 struct mlx5_core_dev *mdev = dev->mdev;
1066
1067 switch (mlx5_get_vport_access_method(ibdev)) {
1068 case MLX5_VPORT_ACCESS_METHOD_MAD:
1069 return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1070
1071 case MLX5_VPORT_ACCESS_METHOD_HCA:
1072 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1073
1074 default:
1075 return -EINVAL;
1076 }
1077
1078 }
1079
1080 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
1081 u16 *pkey)
1082 {
1083 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1084 struct mlx5_core_dev *mdev = dev->mdev;
1085
1086 switch (mlx5_get_vport_access_method(ibdev)) {
1087 case MLX5_VPORT_ACCESS_METHOD_MAD:
1088 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1089
1090 case MLX5_VPORT_ACCESS_METHOD_HCA:
1091 case MLX5_VPORT_ACCESS_METHOD_NIC:
1092 return mlx5_query_hca_vport_pkey(mdev, 0, port, 0, index,
1093 pkey);
1094 default:
1095 return -EINVAL;
1096 }
1097 }
1098
1099 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1100 struct ib_device_modify *props)
1101 {
1102 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1103 struct mlx5_reg_node_desc in;
1104 struct mlx5_reg_node_desc out;
1105 int err;
1106
1107 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1108 return -EOPNOTSUPP;
1109
1110 if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1111 return 0;
1112
1113 /*
1114 * If possible, pass node desc to FW, so it can generate
1115 * a 144 trap. If cmd fails, just ignore.
1116 */
1117 memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1118 err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1119 sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1120 if (err)
1121 return err;
1122
1123 memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1124
1125 return err;
1126 }
1127
1128 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,
1129 u32 value)
1130 {
1131 struct mlx5_hca_vport_context ctx = {};
1132 int err;
1133
1134 err = mlx5_query_hca_vport_context(dev->mdev, 0,
1135 port_num, 0, &ctx);
1136 if (err)
1137 return err;
1138
1139 if (~ctx.cap_mask1_perm & mask) {
1140 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1141 mask, ctx.cap_mask1_perm);
1142 return -EINVAL;
1143 }
1144
1145 ctx.cap_mask1 = value;
1146 ctx.cap_mask1_perm = mask;
1147 err = mlx5_core_modify_hca_vport_context(dev->mdev, 0,
1148 port_num, 0, &ctx);
1149
1150 return err;
1151 }
1152
1153 static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
1154 struct ib_port_modify *props)
1155 {
1156 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1157 struct ib_port_attr attr;
1158 u32 tmp;
1159 int err;
1160 u32 change_mask;
1161 u32 value;
1162 bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1163 IB_LINK_LAYER_INFINIBAND);
1164
1165 /* CM layer calls ib_modify_port() regardless of the link layer. For
1166 * Ethernet ports, qkey violation and Port capabilities are meaningless.
1167 */
1168 if (!is_ib)
1169 return 0;
1170
1171 if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1172 change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1173 value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1174 return set_port_caps_atomic(dev, port, change_mask, value);
1175 }
1176
1177 mutex_lock(&dev->cap_mask_mutex);
1178
1179 err = ib_query_port(ibdev, port, &attr);
1180 if (err)
1181 goto out;
1182
1183 tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1184 ~props->clr_port_cap_mask;
1185
1186 err = mlx5_set_port_caps(dev->mdev, port, tmp);
1187
1188 out:
1189 mutex_unlock(&dev->cap_mask_mutex);
1190 return err;
1191 }
1192
1193 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1194 {
1195 mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1196 caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1197 }
1198
1199 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1200 struct mlx5_ib_alloc_ucontext_req_v2 *req,
1201 u32 *num_sys_pages)
1202 {
1203 int uars_per_sys_page;
1204 int bfregs_per_sys_page;
1205 int ref_bfregs = req->total_num_bfregs;
1206
1207 if (req->total_num_bfregs == 0)
1208 return -EINVAL;
1209
1210 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1211 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1212
1213 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1214 return -ENOMEM;
1215
1216 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1217 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1218 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1219 *num_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1220
1221 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1222 return -EINVAL;
1223
1224 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, using %d sys pages\n",
1225 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1226 lib_uar_4k ? "yes" : "no", ref_bfregs,
1227 req->total_num_bfregs, *num_sys_pages);
1228
1229 return 0;
1230 }
1231
1232 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1233 {
1234 struct mlx5_bfreg_info *bfregi;
1235 int err;
1236 int i;
1237
1238 bfregi = &context->bfregi;
1239 for (i = 0; i < bfregi->num_sys_pages; i++) {
1240 err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1241 if (err)
1242 goto error;
1243
1244 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1245 }
1246 return 0;
1247
1248 error:
1249 for (--i; i >= 0; i--)
1250 if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1251 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1252
1253 return err;
1254 }
1255
1256 static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1257 {
1258 struct mlx5_bfreg_info *bfregi;
1259 int err;
1260 int i;
1261
1262 bfregi = &context->bfregi;
1263 for (i = 0; i < bfregi->num_sys_pages; i++) {
1264 err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1265 if (err) {
1266 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1267 return err;
1268 }
1269 }
1270 return 0;
1271 }
1272
1273 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn)
1274 {
1275 int err;
1276
1277 err = mlx5_core_alloc_transport_domain(dev->mdev, tdn);
1278 if (err)
1279 return err;
1280
1281 if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1282 (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1283 !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1284 return err;
1285
1286 mutex_lock(&dev->lb_mutex);
1287 dev->user_td++;
1288
1289 if (dev->user_td == 2)
1290 err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1291
1292 mutex_unlock(&dev->lb_mutex);
1293 return err;
1294 }
1295
1296 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn)
1297 {
1298 mlx5_core_dealloc_transport_domain(dev->mdev, tdn);
1299
1300 if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1301 (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1302 !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1303 return;
1304
1305 mutex_lock(&dev->lb_mutex);
1306 dev->user_td--;
1307
1308 if (dev->user_td < 2)
1309 mlx5_nic_vport_update_local_lb(dev->mdev, false);
1310
1311 mutex_unlock(&dev->lb_mutex);
1312 }
1313
1314 static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
1315 struct ib_udata *udata)
1316 {
1317 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1318 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1319 struct mlx5_ib_alloc_ucontext_resp resp = {};
1320 struct mlx5_ib_ucontext *context;
1321 struct mlx5_bfreg_info *bfregi;
1322 int ver;
1323 int err;
1324 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1325 max_cqe_version);
1326 bool lib_uar_4k;
1327
1328 if (!dev->ib_active)
1329 return ERR_PTR(-EAGAIN);
1330
1331 if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
1332 ver = 0;
1333 else if (udata->inlen >= min_req_v2)
1334 ver = 2;
1335 else
1336 return ERR_PTR(-EINVAL);
1337
1338 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1339 if (err)
1340 return ERR_PTR(err);
1341
1342 if (req.flags)
1343 return ERR_PTR(-EINVAL);
1344
1345 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1346 return ERR_PTR(-EOPNOTSUPP);
1347
1348 req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1349 MLX5_NON_FP_BFREGS_PER_UAR);
1350 if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1351 return ERR_PTR(-EINVAL);
1352
1353 resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1354 if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
1355 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
1356 resp.cache_line_size = cache_line_size();
1357 resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1358 resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1359 resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1360 resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1361 resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1362 resp.cqe_version = min_t(__u8,
1363 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1364 req.max_cqe_version);
1365 resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1366 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1367 resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1368 MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
1369 resp.response_length = min(offsetof(typeof(resp), response_length) +
1370 sizeof(resp.response_length), udata->outlen);
1371
1372 context = kzalloc(sizeof(*context), GFP_KERNEL);
1373 if (!context)
1374 return ERR_PTR(-ENOMEM);
1375
1376 lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1377 bfregi = &context->bfregi;
1378
1379 /* updates req->total_num_bfregs */
1380 err = calc_total_bfregs(dev, lib_uar_4k, &req, &bfregi->num_sys_pages);
1381 if (err)
1382 goto out_ctx;
1383
1384 mutex_init(&bfregi->lock);
1385 bfregi->lib_uar_4k = lib_uar_4k;
1386 bfregi->count = kcalloc(req.total_num_bfregs, sizeof(*bfregi->count),
1387 GFP_KERNEL);
1388 if (!bfregi->count) {
1389 err = -ENOMEM;
1390 goto out_ctx;
1391 }
1392
1393 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1394 sizeof(*bfregi->sys_pages),
1395 GFP_KERNEL);
1396 if (!bfregi->sys_pages) {
1397 err = -ENOMEM;
1398 goto out_count;
1399 }
1400
1401 err = allocate_uars(dev, context);
1402 if (err)
1403 goto out_sys_pages;
1404
1405 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1406 context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
1407 #endif
1408
1409 context->upd_xlt_page = __get_free_page(GFP_KERNEL);
1410 if (!context->upd_xlt_page) {
1411 err = -ENOMEM;
1412 goto out_uars;
1413 }
1414 mutex_init(&context->upd_xlt_page_mutex);
1415
1416 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
1417 err = mlx5_ib_alloc_transport_domain(dev, &context->tdn);
1418 if (err)
1419 goto out_page;
1420 }
1421
1422 INIT_LIST_HEAD(&context->vma_private_list);
1423 mutex_init(&context->vma_private_list_mutex);
1424 INIT_LIST_HEAD(&context->db_page_list);
1425 mutex_init(&context->db_page_mutex);
1426
1427 resp.tot_bfregs = req.total_num_bfregs;
1428 resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
1429
1430 if (field_avail(typeof(resp), cqe_version, udata->outlen))
1431 resp.response_length += sizeof(resp.cqe_version);
1432
1433 if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
1434 resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1435 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1436 resp.response_length += sizeof(resp.cmds_supp_uhw);
1437 }
1438
1439 if (field_avail(typeof(resp), eth_min_inline, udata->outlen)) {
1440 if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1441 mlx5_query_min_inline(dev->mdev, &resp.eth_min_inline);
1442 resp.eth_min_inline++;
1443 }
1444 resp.response_length += sizeof(resp.eth_min_inline);
1445 }
1446
1447 /*
1448 * We don't want to expose information from the PCI bar that is located
1449 * after 4096 bytes, so if the arch only supports larger pages, let's
1450 * pretend we don't support reading the HCA's core clock. This is also
1451 * forced by mmap function.
1452 */
1453 if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
1454 if (PAGE_SIZE <= 4096) {
1455 resp.comp_mask |=
1456 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1457 resp.hca_core_clock_offset =
1458 offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1459 }
1460 resp.response_length += sizeof(resp.hca_core_clock_offset) +
1461 sizeof(resp.reserved2);
1462 }
1463
1464 if (field_avail(typeof(resp), log_uar_size, udata->outlen))
1465 resp.response_length += sizeof(resp.log_uar_size);
1466
1467 if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
1468 resp.response_length += sizeof(resp.num_uars_per_page);
1469
1470 err = ib_copy_to_udata(udata, &resp, resp.response_length);
1471 if (err)
1472 goto out_td;
1473
1474 bfregi->ver = ver;
1475 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1476 context->cqe_version = resp.cqe_version;
1477 context->lib_caps = req.lib_caps;
1478 print_lib_caps(dev, context->lib_caps);
1479
1480 return &context->ibucontext;
1481
1482 out_td:
1483 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1484 mlx5_ib_dealloc_transport_domain(dev, context->tdn);
1485
1486 out_page:
1487 free_page(context->upd_xlt_page);
1488
1489 out_uars:
1490 deallocate_uars(dev, context);
1491
1492 out_sys_pages:
1493 kfree(bfregi->sys_pages);
1494
1495 out_count:
1496 kfree(bfregi->count);
1497
1498 out_ctx:
1499 kfree(context);
1500
1501 return ERR_PTR(err);
1502 }
1503
1504 static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1505 {
1506 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1507 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1508 struct mlx5_bfreg_info *bfregi;
1509
1510 bfregi = &context->bfregi;
1511 if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1512 mlx5_ib_dealloc_transport_domain(dev, context->tdn);
1513
1514 free_page(context->upd_xlt_page);
1515 deallocate_uars(dev, context);
1516 kfree(bfregi->sys_pages);
1517 kfree(bfregi->count);
1518 kfree(context);
1519
1520 return 0;
1521 }
1522
1523 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1524 struct mlx5_bfreg_info *bfregi,
1525 int idx)
1526 {
1527 int fw_uars_per_page;
1528
1529 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1530
1531 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) +
1532 bfregi->sys_pages[idx] / fw_uars_per_page;
1533 }
1534
1535 static int get_command(unsigned long offset)
1536 {
1537 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
1538 }
1539
1540 static int get_arg(unsigned long offset)
1541 {
1542 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
1543 }
1544
1545 static int get_index(unsigned long offset)
1546 {
1547 return get_arg(offset);
1548 }
1549
1550 static void mlx5_ib_vma_open(struct vm_area_struct *area)
1551 {
1552 /* vma_open is called when a new VMA is created on top of our VMA. This
1553 * is done through either mremap flow or split_vma (usually due to
1554 * mlock, madvise, munmap, etc.) We do not support a clone of the VMA,
1555 * as this VMA is strongly hardware related. Therefore we set the
1556 * vm_ops of the newly created/cloned VMA to NULL, to prevent it from
1557 * calling us again and trying to do incorrect actions. We assume that
1558 * the original VMA size is exactly a single page, and therefore all
1559 * "splitting" operation will not happen to it.
1560 */
1561 area->vm_ops = NULL;
1562 }
1563
1564 static void mlx5_ib_vma_close(struct vm_area_struct *area)
1565 {
1566 struct mlx5_ib_vma_private_data *mlx5_ib_vma_priv_data;
1567
1568 /* It's guaranteed that all VMAs opened on a FD are closed before the
1569 * file itself is closed, therefore no sync is needed with the regular
1570 * closing flow. (e.g. mlx5 ib_dealloc_ucontext)
1571 * However need a sync with accessing the vma as part of
1572 * mlx5_ib_disassociate_ucontext.
1573 * The close operation is usually called under mm->mmap_sem except when
1574 * process is exiting.
1575 * The exiting case is handled explicitly as part of
1576 * mlx5_ib_disassociate_ucontext.
1577 */
1578 mlx5_ib_vma_priv_data = (struct mlx5_ib_vma_private_data *)area->vm_private_data;
1579
1580 /* setting the vma context pointer to null in the mlx5_ib driver's
1581 * private data, to protect a race condition in
1582 * mlx5_ib_disassociate_ucontext().
1583 */
1584 mlx5_ib_vma_priv_data->vma = NULL;
1585 mutex_lock(mlx5_ib_vma_priv_data->vma_private_list_mutex);
1586 list_del(&mlx5_ib_vma_priv_data->list);
1587 mutex_unlock(mlx5_ib_vma_priv_data->vma_private_list_mutex);
1588 kfree(mlx5_ib_vma_priv_data);
1589 }
1590
1591 static const struct vm_operations_struct mlx5_ib_vm_ops = {
1592 .open = mlx5_ib_vma_open,
1593 .close = mlx5_ib_vma_close
1594 };
1595
1596 static int mlx5_ib_set_vma_data(struct vm_area_struct *vma,
1597 struct mlx5_ib_ucontext *ctx)
1598 {
1599 struct mlx5_ib_vma_private_data *vma_prv;
1600 struct list_head *vma_head = &ctx->vma_private_list;
1601
1602 vma_prv = kzalloc(sizeof(*vma_prv), GFP_KERNEL);
1603 if (!vma_prv)
1604 return -ENOMEM;
1605
1606 vma_prv->vma = vma;
1607 vma_prv->vma_private_list_mutex = &ctx->vma_private_list_mutex;
1608 vma->vm_private_data = vma_prv;
1609 vma->vm_ops = &mlx5_ib_vm_ops;
1610
1611 mutex_lock(&ctx->vma_private_list_mutex);
1612 list_add(&vma_prv->list, vma_head);
1613 mutex_unlock(&ctx->vma_private_list_mutex);
1614
1615 return 0;
1616 }
1617
1618 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
1619 {
1620 int ret;
1621 struct vm_area_struct *vma;
1622 struct mlx5_ib_vma_private_data *vma_private, *n;
1623 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1624 struct task_struct *owning_process = NULL;
1625 struct mm_struct *owning_mm = NULL;
1626
1627 owning_process = get_pid_task(ibcontext->tgid, PIDTYPE_PID);
1628 if (!owning_process)
1629 return;
1630
1631 owning_mm = get_task_mm(owning_process);
1632 if (!owning_mm) {
1633 pr_info("no mm, disassociate ucontext is pending task termination\n");
1634 while (1) {
1635 put_task_struct(owning_process);
1636 usleep_range(1000, 2000);
1637 owning_process = get_pid_task(ibcontext->tgid,
1638 PIDTYPE_PID);
1639 if (!owning_process ||
1640 owning_process->state == TASK_DEAD) {
1641 pr_info("disassociate ucontext done, task was terminated\n");
1642 /* in case task was dead need to release the
1643 * task struct.
1644 */
1645 if (owning_process)
1646 put_task_struct(owning_process);
1647 return;
1648 }
1649 }
1650 }
1651
1652 /* need to protect from a race on closing the vma as part of
1653 * mlx5_ib_vma_close.
1654 */
1655 down_write(&owning_mm->mmap_sem);
1656 mutex_lock(&context->vma_private_list_mutex);
1657 list_for_each_entry_safe(vma_private, n, &context->vma_private_list,
1658 list) {
1659 vma = vma_private->vma;
1660 ret = zap_vma_ptes(vma, vma->vm_start,
1661 PAGE_SIZE);
1662 WARN_ONCE(ret, "%s: zap_vma_ptes failed", __func__);
1663 /* context going to be destroyed, should
1664 * not access ops any more.
1665 */
1666 vma->vm_flags &= ~(VM_SHARED | VM_MAYSHARE);
1667 vma->vm_ops = NULL;
1668 list_del(&vma_private->list);
1669 kfree(vma_private);
1670 }
1671 mutex_unlock(&context->vma_private_list_mutex);
1672 up_write(&owning_mm->mmap_sem);
1673 mmput(owning_mm);
1674 put_task_struct(owning_process);
1675 }
1676
1677 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
1678 {
1679 switch (cmd) {
1680 case MLX5_IB_MMAP_WC_PAGE:
1681 return "WC";
1682 case MLX5_IB_MMAP_REGULAR_PAGE:
1683 return "best effort WC";
1684 case MLX5_IB_MMAP_NC_PAGE:
1685 return "NC";
1686 default:
1687 return NULL;
1688 }
1689 }
1690
1691 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
1692 struct vm_area_struct *vma,
1693 struct mlx5_ib_ucontext *context)
1694 {
1695 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1696 int err;
1697 unsigned long idx;
1698 phys_addr_t pfn, pa;
1699 pgprot_t prot;
1700 int uars_per_page;
1701
1702 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1703 return -EINVAL;
1704
1705 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
1706 idx = get_index(vma->vm_pgoff);
1707 if (idx % uars_per_page ||
1708 idx * uars_per_page >= bfregi->num_sys_pages) {
1709 mlx5_ib_warn(dev, "invalid uar index %lu\n", idx);
1710 return -EINVAL;
1711 }
1712
1713 switch (cmd) {
1714 case MLX5_IB_MMAP_WC_PAGE:
1715 /* Some architectures don't support WC memory */
1716 #if defined(CONFIG_X86)
1717 if (!pat_enabled())
1718 return -EPERM;
1719 #elif !(defined(CONFIG_PPC) || (defined(CONFIG_ARM) && defined(CONFIG_MMU)))
1720 return -EPERM;
1721 #endif
1722 /* fall through */
1723 case MLX5_IB_MMAP_REGULAR_PAGE:
1724 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
1725 prot = pgprot_writecombine(vma->vm_page_prot);
1726 break;
1727 case MLX5_IB_MMAP_NC_PAGE:
1728 prot = pgprot_noncached(vma->vm_page_prot);
1729 break;
1730 default:
1731 return -EINVAL;
1732 }
1733
1734 pfn = uar_index2pfn(dev, bfregi, idx);
1735 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
1736
1737 vma->vm_page_prot = prot;
1738 err = io_remap_pfn_range(vma, vma->vm_start, pfn,
1739 PAGE_SIZE, vma->vm_page_prot);
1740 if (err) {
1741 mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
1742 err, vma->vm_start, &pfn, mmap_cmd2str(cmd));
1743 return -EAGAIN;
1744 }
1745
1746 pa = pfn << PAGE_SHIFT;
1747 mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA %pa\n", mmap_cmd2str(cmd),
1748 vma->vm_start, &pa);
1749
1750 return mlx5_ib_set_vma_data(vma, context);
1751 }
1752
1753 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
1754 {
1755 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
1756 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1757 unsigned long command;
1758 phys_addr_t pfn;
1759
1760 command = get_command(vma->vm_pgoff);
1761 switch (command) {
1762 case MLX5_IB_MMAP_WC_PAGE:
1763 case MLX5_IB_MMAP_NC_PAGE:
1764 case MLX5_IB_MMAP_REGULAR_PAGE:
1765 return uar_mmap(dev, command, vma, context);
1766
1767 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
1768 return -ENOSYS;
1769
1770 case MLX5_IB_MMAP_CORE_CLOCK:
1771 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1772 return -EINVAL;
1773
1774 if (vma->vm_flags & VM_WRITE)
1775 return -EPERM;
1776
1777 /* Don't expose to user-space information it shouldn't have */
1778 if (PAGE_SIZE > 4096)
1779 return -EOPNOTSUPP;
1780
1781 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1782 pfn = (dev->mdev->iseg_base +
1783 offsetof(struct mlx5_init_seg, internal_timer_h)) >>
1784 PAGE_SHIFT;
1785 if (io_remap_pfn_range(vma, vma->vm_start, pfn,
1786 PAGE_SIZE, vma->vm_page_prot))
1787 return -EAGAIN;
1788
1789 mlx5_ib_dbg(dev, "mapped internal timer at 0x%lx, PA 0x%llx\n",
1790 vma->vm_start,
1791 (unsigned long long)pfn << PAGE_SHIFT);
1792 break;
1793
1794 default:
1795 return -EINVAL;
1796 }
1797
1798 return 0;
1799 }
1800
1801 static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
1802 struct ib_ucontext *context,
1803 struct ib_udata *udata)
1804 {
1805 struct mlx5_ib_alloc_pd_resp resp;
1806 struct mlx5_ib_pd *pd;
1807 int err;
1808
1809 pd = kmalloc(sizeof(*pd), GFP_KERNEL);
1810 if (!pd)
1811 return ERR_PTR(-ENOMEM);
1812
1813 err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
1814 if (err) {
1815 kfree(pd);
1816 return ERR_PTR(err);
1817 }
1818
1819 if (context) {
1820 resp.pdn = pd->pdn;
1821 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
1822 mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
1823 kfree(pd);
1824 return ERR_PTR(-EFAULT);
1825 }
1826 }
1827
1828 return &pd->ibpd;
1829 }
1830
1831 static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
1832 {
1833 struct mlx5_ib_dev *mdev = to_mdev(pd->device);
1834 struct mlx5_ib_pd *mpd = to_mpd(pd);
1835
1836 mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
1837 kfree(mpd);
1838
1839 return 0;
1840 }
1841
1842 enum {
1843 MATCH_CRITERIA_ENABLE_OUTER_BIT,
1844 MATCH_CRITERIA_ENABLE_MISC_BIT,
1845 MATCH_CRITERIA_ENABLE_INNER_BIT
1846 };
1847
1848 #define HEADER_IS_ZERO(match_criteria, headers) \
1849 !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
1850 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
1851
1852 static u8 get_match_criteria_enable(u32 *match_criteria)
1853 {
1854 u8 match_criteria_enable;
1855
1856 match_criteria_enable =
1857 (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
1858 MATCH_CRITERIA_ENABLE_OUTER_BIT;
1859 match_criteria_enable |=
1860 (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
1861 MATCH_CRITERIA_ENABLE_MISC_BIT;
1862 match_criteria_enable |=
1863 (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
1864 MATCH_CRITERIA_ENABLE_INNER_BIT;
1865
1866 return match_criteria_enable;
1867 }
1868
1869 static void set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
1870 {
1871 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
1872 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
1873 }
1874
1875 static void set_flow_label(void *misc_c, void *misc_v, u8 mask, u8 val,
1876 bool inner)
1877 {
1878 if (inner) {
1879 MLX5_SET(fte_match_set_misc,
1880 misc_c, inner_ipv6_flow_label, mask);
1881 MLX5_SET(fte_match_set_misc,
1882 misc_v, inner_ipv6_flow_label, val);
1883 } else {
1884 MLX5_SET(fte_match_set_misc,
1885 misc_c, outer_ipv6_flow_label, mask);
1886 MLX5_SET(fte_match_set_misc,
1887 misc_v, outer_ipv6_flow_label, val);
1888 }
1889 }
1890
1891 static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
1892 {
1893 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
1894 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
1895 MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
1896 MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
1897 }
1898
1899 #define LAST_ETH_FIELD vlan_tag
1900 #define LAST_IB_FIELD sl
1901 #define LAST_IPV4_FIELD tos
1902 #define LAST_IPV6_FIELD traffic_class
1903 #define LAST_TCP_UDP_FIELD src_port
1904 #define LAST_TUNNEL_FIELD tunnel_id
1905 #define LAST_FLOW_TAG_FIELD tag_id
1906 #define LAST_DROP_FIELD size
1907
1908 /* Field is the last supported field */
1909 #define FIELDS_NOT_SUPPORTED(filter, field)\
1910 memchr_inv((void *)&filter.field +\
1911 sizeof(filter.field), 0,\
1912 sizeof(filter) -\
1913 offsetof(typeof(filter), field) -\
1914 sizeof(filter.field))
1915
1916 #define IPV4_VERSION 4
1917 #define IPV6_VERSION 6
1918 static int parse_flow_attr(struct mlx5_core_dev *mdev, u32 *match_c,
1919 u32 *match_v, const union ib_flow_spec *ib_spec,
1920 u32 *tag_id, bool *is_drop)
1921 {
1922 void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
1923 misc_parameters);
1924 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
1925 misc_parameters);
1926 void *headers_c;
1927 void *headers_v;
1928 int match_ipv;
1929
1930 if (ib_spec->type & IB_FLOW_SPEC_INNER) {
1931 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
1932 inner_headers);
1933 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
1934 inner_headers);
1935 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
1936 ft_field_support.inner_ip_version);
1937 } else {
1938 headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
1939 outer_headers);
1940 headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
1941 outer_headers);
1942 match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
1943 ft_field_support.outer_ip_version);
1944 }
1945
1946 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
1947 case IB_FLOW_SPEC_ETH:
1948 if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
1949 return -EOPNOTSUPP;
1950
1951 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1952 dmac_47_16),
1953 ib_spec->eth.mask.dst_mac);
1954 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1955 dmac_47_16),
1956 ib_spec->eth.val.dst_mac);
1957
1958 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
1959 smac_47_16),
1960 ib_spec->eth.mask.src_mac);
1961 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
1962 smac_47_16),
1963 ib_spec->eth.val.src_mac);
1964
1965 if (ib_spec->eth.mask.vlan_tag) {
1966 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1967 cvlan_tag, 1);
1968 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1969 cvlan_tag, 1);
1970
1971 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1972 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
1973 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1974 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
1975
1976 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1977 first_cfi,
1978 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
1979 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1980 first_cfi,
1981 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
1982
1983 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1984 first_prio,
1985 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
1986 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1987 first_prio,
1988 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
1989 }
1990 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
1991 ethertype, ntohs(ib_spec->eth.mask.ether_type));
1992 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
1993 ethertype, ntohs(ib_spec->eth.val.ether_type));
1994 break;
1995 case IB_FLOW_SPEC_IPV4:
1996 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
1997 return -EOPNOTSUPP;
1998
1999 if (match_ipv) {
2000 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2001 ip_version, 0xf);
2002 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2003 ip_version, IPV4_VERSION);
2004 } else {
2005 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2006 ethertype, 0xffff);
2007 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2008 ethertype, ETH_P_IP);
2009 }
2010
2011 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2012 src_ipv4_src_ipv6.ipv4_layout.ipv4),
2013 &ib_spec->ipv4.mask.src_ip,
2014 sizeof(ib_spec->ipv4.mask.src_ip));
2015 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2016 src_ipv4_src_ipv6.ipv4_layout.ipv4),
2017 &ib_spec->ipv4.val.src_ip,
2018 sizeof(ib_spec->ipv4.val.src_ip));
2019 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2020 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2021 &ib_spec->ipv4.mask.dst_ip,
2022 sizeof(ib_spec->ipv4.mask.dst_ip));
2023 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2024 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2025 &ib_spec->ipv4.val.dst_ip,
2026 sizeof(ib_spec->ipv4.val.dst_ip));
2027
2028 set_tos(headers_c, headers_v,
2029 ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
2030
2031 set_proto(headers_c, headers_v,
2032 ib_spec->ipv4.mask.proto, ib_spec->ipv4.val.proto);
2033 break;
2034 case IB_FLOW_SPEC_IPV6:
2035 if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
2036 return -EOPNOTSUPP;
2037
2038 if (match_ipv) {
2039 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2040 ip_version, 0xf);
2041 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2042 ip_version, IPV6_VERSION);
2043 } else {
2044 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2045 ethertype, 0xffff);
2046 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2047 ethertype, ETH_P_IPV6);
2048 }
2049
2050 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2051 src_ipv4_src_ipv6.ipv6_layout.ipv6),
2052 &ib_spec->ipv6.mask.src_ip,
2053 sizeof(ib_spec->ipv6.mask.src_ip));
2054 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2055 src_ipv4_src_ipv6.ipv6_layout.ipv6),
2056 &ib_spec->ipv6.val.src_ip,
2057 sizeof(ib_spec->ipv6.val.src_ip));
2058 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2059 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2060 &ib_spec->ipv6.mask.dst_ip,
2061 sizeof(ib_spec->ipv6.mask.dst_ip));
2062 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2063 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2064 &ib_spec->ipv6.val.dst_ip,
2065 sizeof(ib_spec->ipv6.val.dst_ip));
2066
2067 set_tos(headers_c, headers_v,
2068 ib_spec->ipv6.mask.traffic_class,
2069 ib_spec->ipv6.val.traffic_class);
2070
2071 set_proto(headers_c, headers_v,
2072 ib_spec->ipv6.mask.next_hdr,
2073 ib_spec->ipv6.val.next_hdr);
2074
2075 set_flow_label(misc_params_c, misc_params_v,
2076 ntohl(ib_spec->ipv6.mask.flow_label),
2077 ntohl(ib_spec->ipv6.val.flow_label),
2078 ib_spec->type & IB_FLOW_SPEC_INNER);
2079
2080 break;
2081 case IB_FLOW_SPEC_TCP:
2082 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2083 LAST_TCP_UDP_FIELD))
2084 return -EOPNOTSUPP;
2085
2086 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2087 0xff);
2088 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2089 IPPROTO_TCP);
2090
2091 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
2092 ntohs(ib_spec->tcp_udp.mask.src_port));
2093 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
2094 ntohs(ib_spec->tcp_udp.val.src_port));
2095
2096 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
2097 ntohs(ib_spec->tcp_udp.mask.dst_port));
2098 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
2099 ntohs(ib_spec->tcp_udp.val.dst_port));
2100 break;
2101 case IB_FLOW_SPEC_UDP:
2102 if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
2103 LAST_TCP_UDP_FIELD))
2104 return -EOPNOTSUPP;
2105
2106 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2107 0xff);
2108 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2109 IPPROTO_UDP);
2110
2111 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
2112 ntohs(ib_spec->tcp_udp.mask.src_port));
2113 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
2114 ntohs(ib_spec->tcp_udp.val.src_port));
2115
2116 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
2117 ntohs(ib_spec->tcp_udp.mask.dst_port));
2118 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
2119 ntohs(ib_spec->tcp_udp.val.dst_port));
2120 break;
2121 case IB_FLOW_SPEC_VXLAN_TUNNEL:
2122 if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
2123 LAST_TUNNEL_FIELD))
2124 return -EOPNOTSUPP;
2125
2126 MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
2127 ntohl(ib_spec->tunnel.mask.tunnel_id));
2128 MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
2129 ntohl(ib_spec->tunnel.val.tunnel_id));
2130 break;
2131 case IB_FLOW_SPEC_ACTION_TAG:
2132 if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
2133 LAST_FLOW_TAG_FIELD))
2134 return -EOPNOTSUPP;
2135 if (ib_spec->flow_tag.tag_id >= BIT(24))
2136 return -EINVAL;
2137
2138 *tag_id = ib_spec->flow_tag.tag_id;
2139 break;
2140 case IB_FLOW_SPEC_ACTION_DROP:
2141 if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
2142 LAST_DROP_FIELD))
2143 return -EOPNOTSUPP;
2144 *is_drop = true;
2145 break;
2146 default:
2147 return -EINVAL;
2148 }
2149
2150 return 0;
2151 }
2152
2153 /* If a flow could catch both multicast and unicast packets,
2154 * it won't fall into the multicast flow steering table and this rule
2155 * could steal other multicast packets.
2156 */
2157 static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
2158 {
2159 union ib_flow_spec *flow_spec;
2160
2161 if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
2162 ib_attr->num_of_specs < 1)
2163 return false;
2164
2165 flow_spec = (union ib_flow_spec *)(ib_attr + 1);
2166 if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
2167 struct ib_flow_spec_ipv4 *ipv4_spec;
2168
2169 ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
2170 if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
2171 return true;
2172
2173 return false;
2174 }
2175
2176 if (flow_spec->type == IB_FLOW_SPEC_ETH) {
2177 struct ib_flow_spec_eth *eth_spec;
2178
2179 eth_spec = (struct ib_flow_spec_eth *)flow_spec;
2180 return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
2181 is_multicast_ether_addr(eth_spec->val.dst_mac);
2182 }
2183
2184 return false;
2185 }
2186
2187 static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
2188 const struct ib_flow_attr *flow_attr,
2189 bool check_inner)
2190 {
2191 union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
2192 int match_ipv = check_inner ?
2193 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2194 ft_field_support.inner_ip_version) :
2195 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2196 ft_field_support.outer_ip_version);
2197 int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
2198 bool ipv4_spec_valid, ipv6_spec_valid;
2199 unsigned int ip_spec_type = 0;
2200 bool has_ethertype = false;
2201 unsigned int spec_index;
2202 bool mask_valid = true;
2203 u16 eth_type = 0;
2204 bool type_valid;
2205
2206 /* Validate that ethertype is correct */
2207 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
2208 if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
2209 ib_spec->eth.mask.ether_type) {
2210 mask_valid = (ib_spec->eth.mask.ether_type ==
2211 htons(0xffff));
2212 has_ethertype = true;
2213 eth_type = ntohs(ib_spec->eth.val.ether_type);
2214 } else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
2215 (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
2216 ip_spec_type = ib_spec->type;
2217 }
2218 ib_spec = (void *)ib_spec + ib_spec->size;
2219 }
2220
2221 type_valid = (!has_ethertype) || (!ip_spec_type);
2222 if (!type_valid && mask_valid) {
2223 ipv4_spec_valid = (eth_type == ETH_P_IP) &&
2224 (ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
2225 ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
2226 (ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
2227
2228 type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
2229 (((eth_type == ETH_P_MPLS_UC) ||
2230 (eth_type == ETH_P_MPLS_MC)) && match_ipv);
2231 }
2232
2233 return type_valid;
2234 }
2235
2236 static bool is_valid_attr(struct mlx5_core_dev *mdev,
2237 const struct ib_flow_attr *flow_attr)
2238 {
2239 return is_valid_ethertype(mdev, flow_attr, false) &&
2240 is_valid_ethertype(mdev, flow_attr, true);
2241 }
2242
2243 static void put_flow_table(struct mlx5_ib_dev *dev,
2244 struct mlx5_ib_flow_prio *prio, bool ft_added)
2245 {
2246 prio->refcount -= !!ft_added;
2247 if (!prio->refcount) {
2248 mlx5_destroy_flow_table(prio->flow_table);
2249 prio->flow_table = NULL;
2250 }
2251 }
2252
2253 static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
2254 {
2255 struct mlx5_ib_dev *dev = to_mdev(flow_id->qp->device);
2256 struct mlx5_ib_flow_handler *handler = container_of(flow_id,
2257 struct mlx5_ib_flow_handler,
2258 ibflow);
2259 struct mlx5_ib_flow_handler *iter, *tmp;
2260
2261 mutex_lock(&dev->flow_db.lock);
2262
2263 list_for_each_entry_safe(iter, tmp, &handler->list, list) {
2264 mlx5_del_flow_rules(iter->rule);
2265 put_flow_table(dev, iter->prio, true);
2266 list_del(&iter->list);
2267 kfree(iter);
2268 }
2269
2270 mlx5_del_flow_rules(handler->rule);
2271 put_flow_table(dev, handler->prio, true);
2272 mutex_unlock(&dev->flow_db.lock);
2273
2274 kfree(handler);
2275
2276 return 0;
2277 }
2278
2279 static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
2280 {
2281 priority *= 2;
2282 if (!dont_trap)
2283 priority++;
2284 return priority;
2285 }
2286
2287 enum flow_table_type {
2288 MLX5_IB_FT_RX,
2289 MLX5_IB_FT_TX
2290 };
2291
2292 #define MLX5_FS_MAX_TYPES 6
2293 #define MLX5_FS_MAX_ENTRIES BIT(16)
2294 static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
2295 struct ib_flow_attr *flow_attr,
2296 enum flow_table_type ft_type)
2297 {
2298 bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
2299 struct mlx5_flow_namespace *ns = NULL;
2300 struct mlx5_ib_flow_prio *prio;
2301 struct mlx5_flow_table *ft;
2302 int max_table_size;
2303 int num_entries;
2304 int num_groups;
2305 int priority;
2306 int err = 0;
2307
2308 max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
2309 log_max_ft_size));
2310 if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
2311 if (flow_is_multicast_only(flow_attr) &&
2312 !dont_trap)
2313 priority = MLX5_IB_FLOW_MCAST_PRIO;
2314 else
2315 priority = ib_prio_to_core_prio(flow_attr->priority,
2316 dont_trap);
2317 ns = mlx5_get_flow_namespace(dev->mdev,
2318 MLX5_FLOW_NAMESPACE_BYPASS);
2319 num_entries = MLX5_FS_MAX_ENTRIES;
2320 num_groups = MLX5_FS_MAX_TYPES;
2321 prio = &dev->flow_db.prios[priority];
2322 } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
2323 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
2324 ns = mlx5_get_flow_namespace(dev->mdev,
2325 MLX5_FLOW_NAMESPACE_LEFTOVERS);
2326 build_leftovers_ft_param(&priority,
2327 &num_entries,
2328 &num_groups);
2329 prio = &dev->flow_db.prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
2330 } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
2331 if (!MLX5_CAP_FLOWTABLE(dev->mdev,
2332 allow_sniffer_and_nic_rx_shared_tir))
2333 return ERR_PTR(-ENOTSUPP);
2334
2335 ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
2336 MLX5_FLOW_NAMESPACE_SNIFFER_RX :
2337 MLX5_FLOW_NAMESPACE_SNIFFER_TX);
2338
2339 prio = &dev->flow_db.sniffer[ft_type];
2340 priority = 0;
2341 num_entries = 1;
2342 num_groups = 1;
2343 }
2344
2345 if (!ns)
2346 return ERR_PTR(-ENOTSUPP);
2347
2348 if (num_entries > max_table_size)
2349 return ERR_PTR(-ENOMEM);
2350
2351 ft = prio->flow_table;
2352 if (!ft) {
2353 ft = mlx5_create_auto_grouped_flow_table(ns, priority,
2354 num_entries,
2355 num_groups,
2356 0, 0);
2357
2358 if (!IS_ERR(ft)) {
2359 prio->refcount = 0;
2360 prio->flow_table = ft;
2361 } else {
2362 err = PTR_ERR(ft);
2363 }
2364 }
2365
2366 return err ? ERR_PTR(err) : prio;
2367 }
2368
2369 static void set_underlay_qp(struct mlx5_ib_dev *dev,
2370 struct mlx5_flow_spec *spec,
2371 u32 underlay_qpn)
2372 {
2373 void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
2374 spec->match_criteria,
2375 misc_parameters);
2376 void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2377 misc_parameters);
2378
2379 if (underlay_qpn &&
2380 MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
2381 ft_field_support.bth_dst_qp)) {
2382 MLX5_SET(fte_match_set_misc,
2383 misc_params_v, bth_dst_qp, underlay_qpn);
2384 MLX5_SET(fte_match_set_misc,
2385 misc_params_c, bth_dst_qp, 0xffffff);
2386 }
2387 }
2388
2389 static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
2390 struct mlx5_ib_flow_prio *ft_prio,
2391 const struct ib_flow_attr *flow_attr,
2392 struct mlx5_flow_destination *dst,
2393 u32 underlay_qpn)
2394 {
2395 struct mlx5_flow_table *ft = ft_prio->flow_table;
2396 struct mlx5_ib_flow_handler *handler;
2397 struct mlx5_flow_act flow_act = {0};
2398 struct mlx5_flow_spec *spec;
2399 struct mlx5_flow_destination *rule_dst = dst;
2400 const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
2401 unsigned int spec_index;
2402 u32 flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
2403 bool is_drop = false;
2404 int err = 0;
2405 int dest_num = 1;
2406
2407 if (!is_valid_attr(dev->mdev, flow_attr))
2408 return ERR_PTR(-EINVAL);
2409
2410 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2411 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2412 if (!handler || !spec) {
2413 err = -ENOMEM;
2414 goto free;
2415 }
2416
2417 INIT_LIST_HEAD(&handler->list);
2418
2419 for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
2420 err = parse_flow_attr(dev->mdev, spec->match_criteria,
2421 spec->match_value,
2422 ib_flow, &flow_tag, &is_drop);
2423 if (err < 0)
2424 goto free;
2425
2426 ib_flow += ((union ib_flow_spec *)ib_flow)->size;
2427 }
2428
2429 if (!flow_is_multicast_only(flow_attr))
2430 set_underlay_qp(dev, spec, underlay_qpn);
2431
2432 spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
2433 if (is_drop) {
2434 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2435 rule_dst = NULL;
2436 dest_num = 0;
2437 } else {
2438 flow_act.action = dst ? MLX5_FLOW_CONTEXT_ACTION_FWD_DEST :
2439 MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
2440 }
2441
2442 if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG &&
2443 (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
2444 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
2445 mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
2446 flow_tag, flow_attr->type);
2447 err = -EINVAL;
2448 goto free;
2449 }
2450 flow_act.flow_tag = flow_tag;
2451 handler->rule = mlx5_add_flow_rules(ft, spec,
2452 &flow_act,
2453 rule_dst, dest_num);
2454
2455 if (IS_ERR(handler->rule)) {
2456 err = PTR_ERR(handler->rule);
2457 goto free;
2458 }
2459
2460 ft_prio->refcount++;
2461 handler->prio = ft_prio;
2462
2463 ft_prio->flow_table = ft;
2464 free:
2465 if (err)
2466 kfree(handler);
2467 kvfree(spec);
2468 return err ? ERR_PTR(err) : handler;
2469 }
2470
2471 static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
2472 struct mlx5_ib_flow_prio *ft_prio,
2473 const struct ib_flow_attr *flow_attr,
2474 struct mlx5_flow_destination *dst)
2475 {
2476 return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0);
2477 }
2478
2479 static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,
2480 struct mlx5_ib_flow_prio *ft_prio,
2481 struct ib_flow_attr *flow_attr,
2482 struct mlx5_flow_destination *dst)
2483 {
2484 struct mlx5_ib_flow_handler *handler_dst = NULL;
2485 struct mlx5_ib_flow_handler *handler = NULL;
2486
2487 handler = create_flow_rule(dev, ft_prio, flow_attr, NULL);
2488 if (!IS_ERR(handler)) {
2489 handler_dst = create_flow_rule(dev, ft_prio,
2490 flow_attr, dst);
2491 if (IS_ERR(handler_dst)) {
2492 mlx5_del_flow_rules(handler->rule);
2493 ft_prio->refcount--;
2494 kfree(handler);
2495 handler = handler_dst;
2496 } else {
2497 list_add(&handler_dst->list, &handler->list);
2498 }
2499 }
2500
2501 return handler;
2502 }
2503 enum {
2504 LEFTOVERS_MC,
2505 LEFTOVERS_UC,
2506 };
2507
2508 static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
2509 struct mlx5_ib_flow_prio *ft_prio,
2510 struct ib_flow_attr *flow_attr,
2511 struct mlx5_flow_destination *dst)
2512 {
2513 struct mlx5_ib_flow_handler *handler_ucast = NULL;
2514 struct mlx5_ib_flow_handler *handler = NULL;
2515
2516 static struct {
2517 struct ib_flow_attr flow_attr;
2518 struct ib_flow_spec_eth eth_flow;
2519 } leftovers_specs[] = {
2520 [LEFTOVERS_MC] = {
2521 .flow_attr = {
2522 .num_of_specs = 1,
2523 .size = sizeof(leftovers_specs[0])
2524 },
2525 .eth_flow = {
2526 .type = IB_FLOW_SPEC_ETH,
2527 .size = sizeof(struct ib_flow_spec_eth),
2528 .mask = {.dst_mac = {0x1} },
2529 .val = {.dst_mac = {0x1} }
2530 }
2531 },
2532 [LEFTOVERS_UC] = {
2533 .flow_attr = {
2534 .num_of_specs = 1,
2535 .size = sizeof(leftovers_specs[0])
2536 },
2537 .eth_flow = {
2538 .type = IB_FLOW_SPEC_ETH,
2539 .size = sizeof(struct ib_flow_spec_eth),
2540 .mask = {.dst_mac = {0x1} },
2541 .val = {.dst_mac = {} }
2542 }
2543 }
2544 };
2545
2546 handler = create_flow_rule(dev, ft_prio,
2547 &leftovers_specs[LEFTOVERS_MC].flow_attr,
2548 dst);
2549 if (!IS_ERR(handler) &&
2550 flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
2551 handler_ucast = create_flow_rule(dev, ft_prio,
2552 &leftovers_specs[LEFTOVERS_UC].flow_attr,
2553 dst);
2554 if (IS_ERR(handler_ucast)) {
2555 mlx5_del_flow_rules(handler->rule);
2556 ft_prio->refcount--;
2557 kfree(handler);
2558 handler = handler_ucast;
2559 } else {
2560 list_add(&handler_ucast->list, &handler->list);
2561 }
2562 }
2563
2564 return handler;
2565 }
2566
2567 static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
2568 struct mlx5_ib_flow_prio *ft_rx,
2569 struct mlx5_ib_flow_prio *ft_tx,
2570 struct mlx5_flow_destination *dst)
2571 {
2572 struct mlx5_ib_flow_handler *handler_rx;
2573 struct mlx5_ib_flow_handler *handler_tx;
2574 int err;
2575 static const struct ib_flow_attr flow_attr = {
2576 .num_of_specs = 0,
2577 .size = sizeof(flow_attr)
2578 };
2579
2580 handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
2581 if (IS_ERR(handler_rx)) {
2582 err = PTR_ERR(handler_rx);
2583 goto err;
2584 }
2585
2586 handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
2587 if (IS_ERR(handler_tx)) {
2588 err = PTR_ERR(handler_tx);
2589 goto err_tx;
2590 }
2591
2592 list_add(&handler_tx->list, &handler_rx->list);
2593
2594 return handler_rx;
2595
2596 err_tx:
2597 mlx5_del_flow_rules(handler_rx->rule);
2598 ft_rx->refcount--;
2599 kfree(handler_rx);
2600 err:
2601 return ERR_PTR(err);
2602 }
2603
2604 static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
2605 struct ib_flow_attr *flow_attr,
2606 int domain)
2607 {
2608 struct mlx5_ib_dev *dev = to_mdev(qp->device);
2609 struct mlx5_ib_qp *mqp = to_mqp(qp);
2610 struct mlx5_ib_flow_handler *handler = NULL;
2611 struct mlx5_flow_destination *dst = NULL;
2612 struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
2613 struct mlx5_ib_flow_prio *ft_prio;
2614 int err;
2615 int underlay_qpn;
2616
2617 if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO)
2618 return ERR_PTR(-ENOMEM);
2619
2620 if (domain != IB_FLOW_DOMAIN_USER ||
2621 flow_attr->port > MLX5_CAP_GEN(dev->mdev, num_ports) ||
2622 (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP))
2623 return ERR_PTR(-EINVAL);
2624
2625 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
2626 if (!dst)
2627 return ERR_PTR(-ENOMEM);
2628
2629 mutex_lock(&dev->flow_db.lock);
2630
2631 ft_prio = get_flow_table(dev, flow_attr, MLX5_IB_FT_RX);
2632 if (IS_ERR(ft_prio)) {
2633 err = PTR_ERR(ft_prio);
2634 goto unlock;
2635 }
2636 if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
2637 ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
2638 if (IS_ERR(ft_prio_tx)) {
2639 err = PTR_ERR(ft_prio_tx);
2640 ft_prio_tx = NULL;
2641 goto destroy_ft;
2642 }
2643 }
2644
2645 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2646 if (mqp->flags & MLX5_IB_QP_RSS)
2647 dst->tir_num = mqp->rss_qp.tirn;
2648 else
2649 dst->tir_num = mqp->raw_packet_qp.rq.tirn;
2650
2651 if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
2652 if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) {
2653 handler = create_dont_trap_rule(dev, ft_prio,
2654 flow_attr, dst);
2655 } else {
2656 underlay_qpn = (mqp->flags & MLX5_IB_QP_UNDERLAY) ?
2657 mqp->underlay_qpn : 0;
2658 handler = _create_flow_rule(dev, ft_prio, flow_attr,
2659 dst, underlay_qpn);
2660 }
2661 } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
2662 flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
2663 handler = create_leftovers_rule(dev, ft_prio, flow_attr,
2664 dst);
2665 } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
2666 handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
2667 } else {
2668 err = -EINVAL;
2669 goto destroy_ft;
2670 }
2671
2672 if (IS_ERR(handler)) {
2673 err = PTR_ERR(handler);
2674 handler = NULL;
2675 goto destroy_ft;
2676 }
2677
2678 mutex_unlock(&dev->flow_db.lock);
2679 kfree(dst);
2680
2681 return &handler->ibflow;
2682
2683 destroy_ft:
2684 put_flow_table(dev, ft_prio, false);
2685 if (ft_prio_tx)
2686 put_flow_table(dev, ft_prio_tx, false);
2687 unlock:
2688 mutex_unlock(&dev->flow_db.lock);
2689 kfree(dst);
2690 kfree(handler);
2691 return ERR_PTR(err);
2692 }
2693
2694 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2695 {
2696 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2697 struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2698 int err;
2699
2700 if (mqp->flags & MLX5_IB_QP_UNDERLAY) {
2701 mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2702 return -EOPNOTSUPP;
2703 }
2704
2705 err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
2706 if (err)
2707 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2708 ibqp->qp_num, gid->raw);
2709
2710 return err;
2711 }
2712
2713 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2714 {
2715 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2716 int err;
2717
2718 err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
2719 if (err)
2720 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2721 ibqp->qp_num, gid->raw);
2722
2723 return err;
2724 }
2725
2726 static int init_node_data(struct mlx5_ib_dev *dev)
2727 {
2728 int err;
2729
2730 err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2731 if (err)
2732 return err;
2733
2734 dev->mdev->rev_id = dev->mdev->pdev->revision;
2735
2736 return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2737 }
2738
2739 static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
2740 char *buf)
2741 {
2742 struct mlx5_ib_dev *dev =
2743 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
2744
2745 return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
2746 }
2747
2748 static ssize_t show_reg_pages(struct device *device,
2749 struct device_attribute *attr, char *buf)
2750 {
2751 struct mlx5_ib_dev *dev =
2752 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
2753
2754 return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2755 }
2756
2757 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
2758 char *buf)
2759 {
2760 struct mlx5_ib_dev *dev =
2761 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
2762 return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
2763 }
2764
2765 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
2766 char *buf)
2767 {
2768 struct mlx5_ib_dev *dev =
2769 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
2770 return sprintf(buf, "%x\n", dev->mdev->rev_id);
2771 }
2772
2773 static ssize_t show_board(struct device *device, struct device_attribute *attr,
2774 char *buf)
2775 {
2776 struct mlx5_ib_dev *dev =
2777 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
2778 return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2779 dev->mdev->board_id);
2780 }
2781
2782 static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
2783 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
2784 static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
2785 static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
2786 static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
2787
2788 static struct device_attribute *mlx5_class_attributes[] = {
2789 &dev_attr_hw_rev,
2790 &dev_attr_hca_type,
2791 &dev_attr_board_id,
2792 &dev_attr_fw_pages,
2793 &dev_attr_reg_pages,
2794 };
2795
2796 static void pkey_change_handler(struct work_struct *work)
2797 {
2798 struct mlx5_ib_port_resources *ports =
2799 container_of(work, struct mlx5_ib_port_resources,
2800 pkey_change_work);
2801
2802 mutex_lock(&ports->devr->mutex);
2803 mlx5_ib_gsi_pkey_change(ports->gsi);
2804 mutex_unlock(&ports->devr->mutex);
2805 }
2806
2807 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2808 {
2809 struct mlx5_ib_qp *mqp;
2810 struct mlx5_ib_cq *send_mcq, *recv_mcq;
2811 struct mlx5_core_cq *mcq;
2812 struct list_head cq_armed_list;
2813 unsigned long flags_qp;
2814 unsigned long flags_cq;
2815 unsigned long flags;
2816
2817 INIT_LIST_HEAD(&cq_armed_list);
2818
2819 /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2820 spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2821 list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2822 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2823 if (mqp->sq.tail != mqp->sq.head) {
2824 send_mcq = to_mcq(mqp->ibqp.send_cq);
2825 spin_lock_irqsave(&send_mcq->lock, flags_cq);
2826 if (send_mcq->mcq.comp &&
2827 mqp->ibqp.send_cq->comp_handler) {
2828 if (!send_mcq->mcq.reset_notify_added) {
2829 send_mcq->mcq.reset_notify_added = 1;
2830 list_add_tail(&send_mcq->mcq.reset_notify,
2831 &cq_armed_list);
2832 }
2833 }
2834 spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2835 }
2836 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2837 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2838 /* no handling is needed for SRQ */
2839 if (!mqp->ibqp.srq) {
2840 if (mqp->rq.tail != mqp->rq.head) {
2841 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2842 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2843 if (recv_mcq->mcq.comp &&
2844 mqp->ibqp.recv_cq->comp_handler) {
2845 if (!recv_mcq->mcq.reset_notify_added) {
2846 recv_mcq->mcq.reset_notify_added = 1;
2847 list_add_tail(&recv_mcq->mcq.reset_notify,
2848 &cq_armed_list);
2849 }
2850 }
2851 spin_unlock_irqrestore(&recv_mcq->lock,
2852 flags_cq);
2853 }
2854 }
2855 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2856 }
2857 /*At that point all inflight post send were put to be executed as of we
2858 * lock/unlock above locks Now need to arm all involved CQs.
2859 */
2860 list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2861 mcq->comp(mcq);
2862 }
2863 spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2864 }
2865
2866 static void delay_drop_handler(struct work_struct *work)
2867 {
2868 int err;
2869 struct mlx5_ib_delay_drop *delay_drop =
2870 container_of(work, struct mlx5_ib_delay_drop,
2871 delay_drop_work);
2872
2873 atomic_inc(&delay_drop->events_cnt);
2874
2875 mutex_lock(&delay_drop->lock);
2876 err = mlx5_core_set_delay_drop(delay_drop->dev->mdev,
2877 delay_drop->timeout);
2878 if (err) {
2879 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2880 delay_drop->timeout);
2881 delay_drop->activate = false;
2882 }
2883 mutex_unlock(&delay_drop->lock);
2884 }
2885
2886 static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
2887 enum mlx5_dev_event event, unsigned long param)
2888 {
2889 struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
2890 struct ib_event ibev;
2891 bool fatal = false;
2892 u8 port = 0;
2893
2894 switch (event) {
2895 case MLX5_DEV_EVENT_SYS_ERROR:
2896 ibev.event = IB_EVENT_DEVICE_FATAL;
2897 mlx5_ib_handle_internal_error(ibdev);
2898 fatal = true;
2899 break;
2900
2901 case MLX5_DEV_EVENT_PORT_UP:
2902 case MLX5_DEV_EVENT_PORT_DOWN:
2903 case MLX5_DEV_EVENT_PORT_INITIALIZED:
2904 port = (u8)param;
2905
2906 /* In RoCE, port up/down events are handled in
2907 * mlx5_netdev_event().
2908 */
2909 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2910 IB_LINK_LAYER_ETHERNET)
2911 return;
2912
2913 ibev.event = (event == MLX5_DEV_EVENT_PORT_UP) ?
2914 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2915 break;
2916
2917 case MLX5_DEV_EVENT_LID_CHANGE:
2918 ibev.event = IB_EVENT_LID_CHANGE;
2919 port = (u8)param;
2920 break;
2921
2922 case MLX5_DEV_EVENT_PKEY_CHANGE:
2923 ibev.event = IB_EVENT_PKEY_CHANGE;
2924 port = (u8)param;
2925
2926 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2927 break;
2928
2929 case MLX5_DEV_EVENT_GUID_CHANGE:
2930 ibev.event = IB_EVENT_GID_CHANGE;
2931 port = (u8)param;
2932 break;
2933
2934 case MLX5_DEV_EVENT_CLIENT_REREG:
2935 ibev.event = IB_EVENT_CLIENT_REREGISTER;
2936 port = (u8)param;
2937 break;
2938 case MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT:
2939 schedule_work(&ibdev->delay_drop.delay_drop_work);
2940 goto out;
2941 default:
2942 goto out;
2943 }
2944
2945 ibev.device = &ibdev->ib_dev;
2946 ibev.element.port_num = port;
2947
2948 if (port < 1 || port > ibdev->num_ports) {
2949 mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
2950 goto out;
2951 }
2952
2953 if (ibdev->ib_active)
2954 ib_dispatch_event(&ibev);
2955
2956 if (fatal)
2957 ibdev->ib_active = false;
2958
2959 out:
2960 return;
2961 }
2962
2963 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
2964 {
2965 struct mlx5_hca_vport_context vport_ctx;
2966 int err;
2967 int port;
2968
2969 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
2970 dev->mdev->port_caps[port - 1].has_smi = false;
2971 if (MLX5_CAP_GEN(dev->mdev, port_type) ==
2972 MLX5_CAP_PORT_TYPE_IB) {
2973 if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
2974 err = mlx5_query_hca_vport_context(dev->mdev, 0,
2975 port, 0,
2976 &vport_ctx);
2977 if (err) {
2978 mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2979 port, err);
2980 return err;
2981 }
2982 dev->mdev->port_caps[port - 1].has_smi =
2983 vport_ctx.has_smi;
2984 } else {
2985 dev->mdev->port_caps[port - 1].has_smi = true;
2986 }
2987 }
2988 }
2989 return 0;
2990 }
2991
2992 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
2993 {
2994 int port;
2995
2996 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++)
2997 mlx5_query_ext_port_caps(dev, port);
2998 }
2999
3000 static int get_port_caps(struct mlx5_ib_dev *dev)
3001 {
3002 struct ib_device_attr *dprops = NULL;
3003 struct ib_port_attr *pprops = NULL;
3004 int err = -ENOMEM;
3005 int port;
3006 struct ib_udata uhw = {.inlen = 0, .outlen = 0};
3007
3008 pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
3009 if (!pprops)
3010 goto out;
3011
3012 dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
3013 if (!dprops)
3014 goto out;
3015
3016 err = set_has_smi_cap(dev);
3017 if (err)
3018 goto out;
3019
3020 err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
3021 if (err) {
3022 mlx5_ib_warn(dev, "query_device failed %d\n", err);
3023 goto out;
3024 }
3025
3026 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
3027 memset(pprops, 0, sizeof(*pprops));
3028 err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
3029 if (err) {
3030 mlx5_ib_warn(dev, "query_port %d failed %d\n",
3031 port, err);
3032 break;
3033 }
3034 dev->mdev->port_caps[port - 1].pkey_table_len =
3035 dprops->max_pkeys;
3036 dev->mdev->port_caps[port - 1].gid_table_len =
3037 pprops->gid_tbl_len;
3038 mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
3039 dprops->max_pkeys, pprops->gid_tbl_len);
3040 }
3041
3042 out:
3043 kfree(pprops);
3044 kfree(dprops);
3045
3046 return err;
3047 }
3048
3049 static void destroy_umrc_res(struct mlx5_ib_dev *dev)
3050 {
3051 int err;
3052
3053 err = mlx5_mr_cache_cleanup(dev);
3054 if (err)
3055 mlx5_ib_warn(dev, "mr cache cleanup failed\n");
3056
3057 mlx5_ib_destroy_qp(dev->umrc.qp);
3058 ib_free_cq(dev->umrc.cq);
3059 ib_dealloc_pd(dev->umrc.pd);
3060 }
3061
3062 enum {
3063 MAX_UMR_WR = 128,
3064 };
3065
3066 static int create_umr_res(struct mlx5_ib_dev *dev)
3067 {
3068 struct ib_qp_init_attr *init_attr = NULL;
3069 struct ib_qp_attr *attr = NULL;
3070 struct ib_pd *pd;
3071 struct ib_cq *cq;
3072 struct ib_qp *qp;
3073 int ret;
3074
3075 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
3076 init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
3077 if (!attr || !init_attr) {
3078 ret = -ENOMEM;
3079 goto error_0;
3080 }
3081
3082 pd = ib_alloc_pd(&dev->ib_dev, 0);
3083 if (IS_ERR(pd)) {
3084 mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
3085 ret = PTR_ERR(pd);
3086 goto error_0;
3087 }
3088
3089 cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
3090 if (IS_ERR(cq)) {
3091 mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
3092 ret = PTR_ERR(cq);
3093 goto error_2;
3094 }
3095
3096 init_attr->send_cq = cq;
3097 init_attr->recv_cq = cq;
3098 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
3099 init_attr->cap.max_send_wr = MAX_UMR_WR;
3100 init_attr->cap.max_send_sge = 1;
3101 init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
3102 init_attr->port_num = 1;
3103 qp = mlx5_ib_create_qp(pd, init_attr, NULL);
3104 if (IS_ERR(qp)) {
3105 mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
3106 ret = PTR_ERR(qp);
3107 goto error_3;
3108 }
3109 qp->device = &dev->ib_dev;
3110 qp->real_qp = qp;
3111 qp->uobject = NULL;
3112 qp->qp_type = MLX5_IB_QPT_REG_UMR;
3113 qp->send_cq = init_attr->send_cq;
3114 qp->recv_cq = init_attr->recv_cq;
3115
3116 attr->qp_state = IB_QPS_INIT;
3117 attr->port_num = 1;
3118 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
3119 IB_QP_PORT, NULL);
3120 if (ret) {
3121 mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
3122 goto error_4;
3123 }
3124
3125 memset(attr, 0, sizeof(*attr));
3126 attr->qp_state = IB_QPS_RTR;
3127 attr->path_mtu = IB_MTU_256;
3128
3129 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
3130 if (ret) {
3131 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
3132 goto error_4;
3133 }
3134
3135 memset(attr, 0, sizeof(*attr));
3136 attr->qp_state = IB_QPS_RTS;
3137 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
3138 if (ret) {
3139 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
3140 goto error_4;
3141 }
3142
3143 dev->umrc.qp = qp;
3144 dev->umrc.cq = cq;
3145 dev->umrc.pd = pd;
3146
3147 sema_init(&dev->umrc.sem, MAX_UMR_WR);
3148 ret = mlx5_mr_cache_init(dev);
3149 if (ret) {
3150 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
3151 goto error_4;
3152 }
3153
3154 kfree(attr);
3155 kfree(init_attr);
3156
3157 return 0;
3158
3159 error_4:
3160 mlx5_ib_destroy_qp(qp);
3161
3162 error_3:
3163 ib_free_cq(cq);
3164
3165 error_2:
3166 ib_dealloc_pd(pd);
3167
3168 error_0:
3169 kfree(attr);
3170 kfree(init_attr);
3171 return ret;
3172 }
3173
3174 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
3175 {
3176 switch (umr_fence_cap) {
3177 case MLX5_CAP_UMR_FENCE_NONE:
3178 return MLX5_FENCE_MODE_NONE;
3179 case MLX5_CAP_UMR_FENCE_SMALL:
3180 return MLX5_FENCE_MODE_INITIATOR_SMALL;
3181 default:
3182 return MLX5_FENCE_MODE_STRONG_ORDERING;
3183 }
3184 }
3185
3186 static int create_dev_resources(struct mlx5_ib_resources *devr)
3187 {
3188 struct ib_srq_init_attr attr;
3189 struct mlx5_ib_dev *dev;
3190 struct ib_cq_init_attr cq_attr = {.cqe = 1};
3191 int port;
3192 int ret = 0;
3193
3194 dev = container_of(devr, struct mlx5_ib_dev, devr);
3195
3196 mutex_init(&devr->mutex);
3197
3198 devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
3199 if (IS_ERR(devr->p0)) {
3200 ret = PTR_ERR(devr->p0);
3201 goto error0;
3202 }
3203 devr->p0->device = &dev->ib_dev;
3204 devr->p0->uobject = NULL;
3205 atomic_set(&devr->p0->usecnt, 0);
3206
3207 devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL, NULL);
3208 if (IS_ERR(devr->c0)) {
3209 ret = PTR_ERR(devr->c0);
3210 goto error1;
3211 }
3212 devr->c0->device = &dev->ib_dev;
3213 devr->c0->uobject = NULL;
3214 devr->c0->comp_handler = NULL;
3215 devr->c0->event_handler = NULL;
3216 devr->c0->cq_context = NULL;
3217 atomic_set(&devr->c0->usecnt, 0);
3218
3219 devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
3220 if (IS_ERR(devr->x0)) {
3221 ret = PTR_ERR(devr->x0);
3222 goto error2;
3223 }
3224 devr->x0->device = &dev->ib_dev;
3225 devr->x0->inode = NULL;
3226 atomic_set(&devr->x0->usecnt, 0);
3227 mutex_init(&devr->x0->tgt_qp_mutex);
3228 INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
3229
3230 devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
3231 if (IS_ERR(devr->x1)) {
3232 ret = PTR_ERR(devr->x1);
3233 goto error3;
3234 }
3235 devr->x1->device = &dev->ib_dev;
3236 devr->x1->inode = NULL;
3237 atomic_set(&devr->x1->usecnt, 0);
3238 mutex_init(&devr->x1->tgt_qp_mutex);
3239 INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
3240
3241 memset(&attr, 0, sizeof(attr));
3242 attr.attr.max_sge = 1;
3243 attr.attr.max_wr = 1;
3244 attr.srq_type = IB_SRQT_XRC;
3245 attr.ext.cq = devr->c0;
3246 attr.ext.xrc.xrcd = devr->x0;
3247
3248 devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
3249 if (IS_ERR(devr->s0)) {
3250 ret = PTR_ERR(devr->s0);
3251 goto error4;
3252 }
3253 devr->s0->device = &dev->ib_dev;
3254 devr->s0->pd = devr->p0;
3255 devr->s0->uobject = NULL;
3256 devr->s0->event_handler = NULL;
3257 devr->s0->srq_context = NULL;
3258 devr->s0->srq_type = IB_SRQT_XRC;
3259 devr->s0->ext.xrc.xrcd = devr->x0;
3260 devr->s0->ext.cq = devr->c0;
3261 atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
3262 atomic_inc(&devr->s0->ext.cq->usecnt);
3263 atomic_inc(&devr->p0->usecnt);
3264 atomic_set(&devr->s0->usecnt, 0);
3265
3266 memset(&attr, 0, sizeof(attr));
3267 attr.attr.max_sge = 1;
3268 attr.attr.max_wr = 1;
3269 attr.srq_type = IB_SRQT_BASIC;
3270 devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
3271 if (IS_ERR(devr->s1)) {
3272 ret = PTR_ERR(devr->s1);
3273 goto error5;
3274 }
3275 devr->s1->device = &dev->ib_dev;
3276 devr->s1->pd = devr->p0;
3277 devr->s1->uobject = NULL;
3278 devr->s1->event_handler = NULL;
3279 devr->s1->srq_context = NULL;
3280 devr->s1->srq_type = IB_SRQT_BASIC;
3281 devr->s1->ext.cq = devr->c0;
3282 atomic_inc(&devr->p0->usecnt);
3283 atomic_set(&devr->s1->usecnt, 0);
3284
3285 for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
3286 INIT_WORK(&devr->ports[port].pkey_change_work,
3287 pkey_change_handler);
3288 devr->ports[port].devr = devr;
3289 }
3290
3291 return 0;
3292
3293 error5:
3294 mlx5_ib_destroy_srq(devr->s0);
3295 error4:
3296 mlx5_ib_dealloc_xrcd(devr->x1);
3297 error3:
3298 mlx5_ib_dealloc_xrcd(devr->x0);
3299 error2:
3300 mlx5_ib_destroy_cq(devr->c0);
3301 error1:
3302 mlx5_ib_dealloc_pd(devr->p0);
3303 error0:
3304 return ret;
3305 }
3306
3307 static void destroy_dev_resources(struct mlx5_ib_resources *devr)
3308 {
3309 struct mlx5_ib_dev *dev =
3310 container_of(devr, struct mlx5_ib_dev, devr);
3311 int port;
3312
3313 mlx5_ib_destroy_srq(devr->s1);
3314 mlx5_ib_destroy_srq(devr->s0);
3315 mlx5_ib_dealloc_xrcd(devr->x0);
3316 mlx5_ib_dealloc_xrcd(devr->x1);
3317 mlx5_ib_destroy_cq(devr->c0);
3318 mlx5_ib_dealloc_pd(devr->p0);
3319
3320 /* Make sure no change P_Key work items are still executing */
3321 for (port = 0; port < dev->num_ports; ++port)
3322 cancel_work_sync(&devr->ports[port].pkey_change_work);
3323 }
3324
3325 static u32 get_core_cap_flags(struct ib_device *ibdev)
3326 {
3327 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3328 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3329 u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3330 u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3331 u32 ret = 0;
3332
3333 if (ll == IB_LINK_LAYER_INFINIBAND)
3334 return RDMA_CORE_PORT_IBA_IB;
3335
3336 ret = RDMA_CORE_PORT_RAW_PACKET;
3337
3338 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
3339 return ret;
3340
3341 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
3342 return ret;
3343
3344 if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
3345 ret |= RDMA_CORE_PORT_IBA_ROCE;
3346
3347 if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
3348 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3349
3350 return ret;
3351 }
3352
3353 static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
3354 struct ib_port_immutable *immutable)
3355 {
3356 struct ib_port_attr attr;
3357 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3358 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
3359 int err;
3360
3361 immutable->core_cap_flags = get_core_cap_flags(ibdev);
3362
3363 err = ib_query_port(ibdev, port_num, &attr);
3364 if (err)
3365 return err;
3366
3367 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3368 immutable->gid_tbl_len = attr.gid_tbl_len;
3369 immutable->core_cap_flags = get_core_cap_flags(ibdev);
3370 if ((ll == IB_LINK_LAYER_INFINIBAND) || MLX5_CAP_GEN(dev->mdev, roce))
3371 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3372
3373 return 0;
3374 }
3375
3376 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
3377 {
3378 struct mlx5_ib_dev *dev =
3379 container_of(ibdev, struct mlx5_ib_dev, ib_dev);
3380 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
3381 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
3382 fw_rev_sub(dev->mdev));
3383 }
3384
3385 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
3386 {
3387 struct mlx5_core_dev *mdev = dev->mdev;
3388 struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
3389 MLX5_FLOW_NAMESPACE_LAG);
3390 struct mlx5_flow_table *ft;
3391 int err;
3392
3393 if (!ns || !mlx5_lag_is_active(mdev))
3394 return 0;
3395
3396 err = mlx5_cmd_create_vport_lag(mdev);
3397 if (err)
3398 return err;
3399
3400 ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
3401 if (IS_ERR(ft)) {
3402 err = PTR_ERR(ft);
3403 goto err_destroy_vport_lag;
3404 }
3405
3406 dev->flow_db.lag_demux_ft = ft;
3407 return 0;
3408
3409 err_destroy_vport_lag:
3410 mlx5_cmd_destroy_vport_lag(mdev);
3411 return err;
3412 }
3413
3414 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
3415 {
3416 struct mlx5_core_dev *mdev = dev->mdev;
3417
3418 if (dev->flow_db.lag_demux_ft) {
3419 mlx5_destroy_flow_table(dev->flow_db.lag_demux_ft);
3420 dev->flow_db.lag_demux_ft = NULL;
3421
3422 mlx5_cmd_destroy_vport_lag(mdev);
3423 }
3424 }
3425
3426 static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev)
3427 {
3428 int err;
3429
3430 dev->roce.nb.notifier_call = mlx5_netdev_event;
3431 err = register_netdevice_notifier(&dev->roce.nb);
3432 if (err) {
3433 dev->roce.nb.notifier_call = NULL;
3434 return err;
3435 }
3436
3437 return 0;
3438 }
3439
3440 static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev)
3441 {
3442 if (dev->roce.nb.notifier_call) {
3443 unregister_netdevice_notifier(&dev->roce.nb);
3444 dev->roce.nb.notifier_call = NULL;
3445 }
3446 }
3447
3448 static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3449 {
3450 int err;
3451
3452 err = mlx5_add_netdev_notifier(dev);
3453 if (err)
3454 return err;
3455
3456 if (MLX5_CAP_GEN(dev->mdev, roce)) {
3457 err = mlx5_nic_vport_enable_roce(dev->mdev);
3458 if (err)
3459 goto err_unregister_netdevice_notifier;
3460 }
3461
3462 err = mlx5_eth_lag_init(dev);
3463 if (err)
3464 goto err_disable_roce;
3465
3466 return 0;
3467
3468 err_disable_roce:
3469 if (MLX5_CAP_GEN(dev->mdev, roce))
3470 mlx5_nic_vport_disable_roce(dev->mdev);
3471
3472 err_unregister_netdevice_notifier:
3473 mlx5_remove_netdev_notifier(dev);
3474 return err;
3475 }
3476
3477 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3478 {
3479 mlx5_eth_lag_cleanup(dev);
3480 if (MLX5_CAP_GEN(dev->mdev, roce))
3481 mlx5_nic_vport_disable_roce(dev->mdev);
3482 }
3483
3484 struct mlx5_ib_counter {
3485 const char *name;
3486 size_t offset;
3487 };
3488
3489 #define INIT_Q_COUNTER(_name) \
3490 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
3491
3492 static const struct mlx5_ib_counter basic_q_cnts[] = {
3493 INIT_Q_COUNTER(rx_write_requests),
3494 INIT_Q_COUNTER(rx_read_requests),
3495 INIT_Q_COUNTER(rx_atomic_requests),
3496 INIT_Q_COUNTER(out_of_buffer),
3497 };
3498
3499 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
3500 INIT_Q_COUNTER(out_of_sequence),
3501 };
3502
3503 static const struct mlx5_ib_counter retrans_q_cnts[] = {
3504 INIT_Q_COUNTER(duplicate_request),
3505 INIT_Q_COUNTER(rnr_nak_retry_err),
3506 INIT_Q_COUNTER(packet_seq_err),
3507 INIT_Q_COUNTER(implied_nak_seq_err),
3508 INIT_Q_COUNTER(local_ack_timeout_err),
3509 };
3510
3511 #define INIT_CONG_COUNTER(_name) \
3512 { .name = #_name, .offset = \
3513 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
3514
3515 static const struct mlx5_ib_counter cong_cnts[] = {
3516 INIT_CONG_COUNTER(rp_cnp_ignored),
3517 INIT_CONG_COUNTER(rp_cnp_handled),
3518 INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
3519 INIT_CONG_COUNTER(np_cnp_sent),
3520 };
3521
3522 static const struct mlx5_ib_counter extended_err_cnts[] = {
3523 INIT_Q_COUNTER(resp_local_length_error),
3524 INIT_Q_COUNTER(resp_cqe_error),
3525 INIT_Q_COUNTER(req_cqe_error),
3526 INIT_Q_COUNTER(req_remote_invalid_request),
3527 INIT_Q_COUNTER(req_remote_access_errors),
3528 INIT_Q_COUNTER(resp_remote_access_errors),
3529 INIT_Q_COUNTER(resp_cqe_flush_error),
3530 INIT_Q_COUNTER(req_cqe_flush_error),
3531 };
3532
3533 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
3534 {
3535 unsigned int i;
3536
3537 for (i = 0; i < dev->num_ports; i++) {
3538 mlx5_core_dealloc_q_counter(dev->mdev,
3539 dev->port[i].cnts.set_id);
3540 kfree(dev->port[i].cnts.names);
3541 kfree(dev->port[i].cnts.offsets);
3542 }
3543 }
3544
3545 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
3546 struct mlx5_ib_counters *cnts)
3547 {
3548 u32 num_counters;
3549
3550 num_counters = ARRAY_SIZE(basic_q_cnts);
3551
3552 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
3553 num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
3554
3555 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
3556 num_counters += ARRAY_SIZE(retrans_q_cnts);
3557
3558 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
3559 num_counters += ARRAY_SIZE(extended_err_cnts);
3560
3561 cnts->num_q_counters = num_counters;
3562
3563 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
3564 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
3565 num_counters += ARRAY_SIZE(cong_cnts);
3566 }
3567
3568 cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL);
3569 if (!cnts->names)
3570 return -ENOMEM;
3571
3572 cnts->offsets = kcalloc(num_counters,
3573 sizeof(cnts->offsets), GFP_KERNEL);
3574 if (!cnts->offsets)
3575 goto err_names;
3576
3577 return 0;
3578
3579 err_names:
3580 kfree(cnts->names);
3581 return -ENOMEM;
3582 }
3583
3584 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
3585 const char **names,
3586 size_t *offsets)
3587 {
3588 int i;
3589 int j = 0;
3590
3591 for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
3592 names[j] = basic_q_cnts[i].name;
3593 offsets[j] = basic_q_cnts[i].offset;
3594 }
3595
3596 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
3597 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
3598 names[j] = out_of_seq_q_cnts[i].name;
3599 offsets[j] = out_of_seq_q_cnts[i].offset;
3600 }
3601 }
3602
3603 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
3604 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
3605 names[j] = retrans_q_cnts[i].name;
3606 offsets[j] = retrans_q_cnts[i].offset;
3607 }
3608 }
3609
3610 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
3611 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
3612 names[j] = extended_err_cnts[i].name;
3613 offsets[j] = extended_err_cnts[i].offset;
3614 }
3615 }
3616
3617 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
3618 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
3619 names[j] = cong_cnts[i].name;
3620 offsets[j] = cong_cnts[i].offset;
3621 }
3622 }
3623 }
3624
3625 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
3626 {
3627 int i;
3628 int ret;
3629
3630 for (i = 0; i < dev->num_ports; i++) {
3631 struct mlx5_ib_port *port = &dev->port[i];
3632
3633 ret = mlx5_core_alloc_q_counter(dev->mdev,
3634 &port->cnts.set_id);
3635 if (ret) {
3636 mlx5_ib_warn(dev,
3637 "couldn't allocate queue counter for port %d, err %d\n",
3638 i + 1, ret);
3639 goto dealloc_counters;
3640 }
3641
3642 ret = __mlx5_ib_alloc_counters(dev, &port->cnts);
3643 if (ret)
3644 goto dealloc_counters;
3645
3646 mlx5_ib_fill_counters(dev, port->cnts.names,
3647 port->cnts.offsets);
3648 }
3649
3650 return 0;
3651
3652 dealloc_counters:
3653 while (--i >= 0)
3654 mlx5_core_dealloc_q_counter(dev->mdev,
3655 dev->port[i].cnts.set_id);
3656
3657 return ret;
3658 }
3659
3660 static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
3661 u8 port_num)
3662 {
3663 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3664 struct mlx5_ib_port *port = &dev->port[port_num - 1];
3665
3666 /* We support only per port stats */
3667 if (port_num == 0)
3668 return NULL;
3669
3670 return rdma_alloc_hw_stats_struct(port->cnts.names,
3671 port->cnts.num_q_counters +
3672 port->cnts.num_cong_counters,
3673 RDMA_HW_STATS_DEFAULT_LIFESPAN);
3674 }
3675
3676 static int mlx5_ib_query_q_counters(struct mlx5_ib_dev *dev,
3677 struct mlx5_ib_port *port,
3678 struct rdma_hw_stats *stats)
3679 {
3680 int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
3681 void *out;
3682 __be32 val;
3683 int ret, i;
3684
3685 out = kvzalloc(outlen, GFP_KERNEL);
3686 if (!out)
3687 return -ENOMEM;
3688
3689 ret = mlx5_core_query_q_counter(dev->mdev,
3690 port->cnts.set_id, 0,
3691 out, outlen);
3692 if (ret)
3693 goto free;
3694
3695 for (i = 0; i < port->cnts.num_q_counters; i++) {
3696 val = *(__be32 *)(out + port->cnts.offsets[i]);
3697 stats->value[i] = (u64)be32_to_cpu(val);
3698 }
3699
3700 free:
3701 kvfree(out);
3702 return ret;
3703 }
3704
3705 static int mlx5_ib_query_cong_counters(struct mlx5_ib_dev *dev,
3706 struct mlx5_ib_port *port,
3707 struct rdma_hw_stats *stats)
3708 {
3709 int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out);
3710 void *out;
3711 int ret, i;
3712 int offset = port->cnts.num_q_counters;
3713
3714 out = kvzalloc(outlen, GFP_KERNEL);
3715 if (!out)
3716 return -ENOMEM;
3717
3718 ret = mlx5_cmd_query_cong_counter(dev->mdev, false, out, outlen);
3719 if (ret)
3720 goto free;
3721
3722 for (i = 0; i < port->cnts.num_cong_counters; i++) {
3723 stats->value[i + offset] =
3724 be64_to_cpup((__be64 *)(out +
3725 port->cnts.offsets[i + offset]));
3726 }
3727
3728 free:
3729 kvfree(out);
3730 return ret;
3731 }
3732
3733 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
3734 struct rdma_hw_stats *stats,
3735 u8 port_num, int index)
3736 {
3737 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3738 struct mlx5_ib_port *port = &dev->port[port_num - 1];
3739 int ret, num_counters;
3740
3741 if (!stats)
3742 return -EINVAL;
3743
3744 ret = mlx5_ib_query_q_counters(dev, port, stats);
3745 if (ret)
3746 return ret;
3747 num_counters = port->cnts.num_q_counters;
3748
3749 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
3750 ret = mlx5_ib_query_cong_counters(dev, port, stats);
3751 if (ret)
3752 return ret;
3753 num_counters += port->cnts.num_cong_counters;
3754 }
3755
3756 return num_counters;
3757 }
3758
3759 static void mlx5_ib_free_rdma_netdev(struct net_device *netdev)
3760 {
3761 return mlx5_rdma_netdev_free(netdev);
3762 }
3763
3764 static struct net_device*
3765 mlx5_ib_alloc_rdma_netdev(struct ib_device *hca,
3766 u8 port_num,
3767 enum rdma_netdev_t type,
3768 const char *name,
3769 unsigned char name_assign_type,
3770 void (*setup)(struct net_device *))
3771 {
3772 struct net_device *netdev;
3773 struct rdma_netdev *rn;
3774
3775 if (type != RDMA_NETDEV_IPOIB)
3776 return ERR_PTR(-EOPNOTSUPP);
3777
3778 netdev = mlx5_rdma_netdev_alloc(to_mdev(hca)->mdev, hca,
3779 name, setup);
3780 if (likely(!IS_ERR_OR_NULL(netdev))) {
3781 rn = netdev_priv(netdev);
3782 rn->free_rdma_netdev = mlx5_ib_free_rdma_netdev;
3783 }
3784 return netdev;
3785 }
3786
3787 static void delay_drop_debugfs_cleanup(struct mlx5_ib_dev *dev)
3788 {
3789 if (!dev->delay_drop.dbg)
3790 return;
3791 debugfs_remove_recursive(dev->delay_drop.dbg->dir_debugfs);
3792 kfree(dev->delay_drop.dbg);
3793 dev->delay_drop.dbg = NULL;
3794 }
3795
3796 static void cancel_delay_drop(struct mlx5_ib_dev *dev)
3797 {
3798 if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
3799 return;
3800
3801 cancel_work_sync(&dev->delay_drop.delay_drop_work);
3802 delay_drop_debugfs_cleanup(dev);
3803 }
3804
3805 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3806 size_t count, loff_t *pos)
3807 {
3808 struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3809 char lbuf[20];
3810 int len;
3811
3812 len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3813 return simple_read_from_buffer(buf, count, pos, lbuf, len);
3814 }
3815
3816 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3817 size_t count, loff_t *pos)
3818 {
3819 struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3820 u32 timeout;
3821 u32 var;
3822
3823 if (kstrtouint_from_user(buf, count, 0, &var))
3824 return -EFAULT;
3825
3826 timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3827 1000);
3828 if (timeout != var)
3829 mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3830 timeout);
3831
3832 delay_drop->timeout = timeout;
3833
3834 return count;
3835 }
3836
3837 static const struct file_operations fops_delay_drop_timeout = {
3838 .owner = THIS_MODULE,
3839 .open = simple_open,
3840 .write = delay_drop_timeout_write,
3841 .read = delay_drop_timeout_read,
3842 };
3843
3844 static int delay_drop_debugfs_init(struct mlx5_ib_dev *dev)
3845 {
3846 struct mlx5_ib_dbg_delay_drop *dbg;
3847
3848 if (!mlx5_debugfs_root)
3849 return 0;
3850
3851 dbg = kzalloc(sizeof(*dbg), GFP_KERNEL);
3852 if (!dbg)
3853 return -ENOMEM;
3854
3855 dev->delay_drop.dbg = dbg;
3856
3857 dbg->dir_debugfs =
3858 debugfs_create_dir("delay_drop",
3859 dev->mdev->priv.dbg_root);
3860 if (!dbg->dir_debugfs)
3861 goto out_debugfs;
3862
3863 dbg->events_cnt_debugfs =
3864 debugfs_create_atomic_t("num_timeout_events", 0400,
3865 dbg->dir_debugfs,
3866 &dev->delay_drop.events_cnt);
3867 if (!dbg->events_cnt_debugfs)
3868 goto out_debugfs;
3869
3870 dbg->rqs_cnt_debugfs =
3871 debugfs_create_atomic_t("num_rqs", 0400,
3872 dbg->dir_debugfs,
3873 &dev->delay_drop.rqs_cnt);
3874 if (!dbg->rqs_cnt_debugfs)
3875 goto out_debugfs;
3876
3877 dbg->timeout_debugfs =
3878 debugfs_create_file("timeout", 0600,
3879 dbg->dir_debugfs,
3880 &dev->delay_drop,
3881 &fops_delay_drop_timeout);
3882 if (!dbg->timeout_debugfs)
3883 goto out_debugfs;
3884
3885 return 0;
3886
3887 out_debugfs:
3888 delay_drop_debugfs_cleanup(dev);
3889 return -ENOMEM;
3890 }
3891
3892 static void init_delay_drop(struct mlx5_ib_dev *dev)
3893 {
3894 if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
3895 return;
3896
3897 mutex_init(&dev->delay_drop.lock);
3898 dev->delay_drop.dev = dev;
3899 dev->delay_drop.activate = false;
3900 dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
3901 INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
3902 atomic_set(&dev->delay_drop.rqs_cnt, 0);
3903 atomic_set(&dev->delay_drop.events_cnt, 0);
3904
3905 if (delay_drop_debugfs_init(dev))
3906 mlx5_ib_warn(dev, "Failed to init delay drop debugfs\n");
3907 }
3908
3909 static const struct cpumask *
3910 mlx5_ib_get_vector_affinity(struct ib_device *ibdev, int comp_vector)
3911 {
3912 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3913
3914 return mlx5_get_vector_affinity_hint(dev->mdev, comp_vector);
3915 }
3916
3917 static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
3918 {
3919 struct mlx5_ib_dev *dev;
3920 enum rdma_link_layer ll;
3921 int port_type_cap;
3922 const char *name;
3923 int err;
3924 int i;
3925
3926 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
3927 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
3928
3929 printk_once(KERN_INFO "%s", mlx5_version);
3930
3931 dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
3932 if (!dev)
3933 return NULL;
3934
3935 dev->mdev = mdev;
3936
3937 dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port),
3938 GFP_KERNEL);
3939 if (!dev->port)
3940 goto err_dealloc;
3941
3942 rwlock_init(&dev->roce.netdev_lock);
3943 err = get_port_caps(dev);
3944 if (err)
3945 goto err_free_port;
3946
3947 if (mlx5_use_mad_ifc(dev))
3948 get_ext_port_caps(dev);
3949
3950 if (!mlx5_lag_is_active(mdev))
3951 name = "mlx5_%d";
3952 else
3953 name = "mlx5_bond_%d";
3954
3955 strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
3956 dev->ib_dev.owner = THIS_MODULE;
3957 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
3958 dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
3959 dev->num_ports = MLX5_CAP_GEN(mdev, num_ports);
3960 dev->ib_dev.phys_port_cnt = dev->num_ports;
3961 dev->ib_dev.num_comp_vectors =
3962 dev->mdev->priv.eq_table.num_comp_vectors;
3963 dev->ib_dev.dev.parent = &mdev->pdev->dev;
3964
3965 dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
3966 dev->ib_dev.uverbs_cmd_mask =
3967 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
3968 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
3969 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
3970 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
3971 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
3972 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
3973 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
3974 (1ull << IB_USER_VERBS_CMD_REG_MR) |
3975 (1ull << IB_USER_VERBS_CMD_REREG_MR) |
3976 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
3977 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
3978 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
3979 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
3980 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
3981 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
3982 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
3983 (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
3984 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
3985 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
3986 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
3987 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
3988 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
3989 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
3990 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
3991 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
3992 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
3993 dev->ib_dev.uverbs_ex_cmd_mask =
3994 (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE) |
3995 (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) |
3996 (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP) |
3997 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP);
3998
3999 dev->ib_dev.query_device = mlx5_ib_query_device;
4000 dev->ib_dev.query_port = mlx5_ib_query_port;
4001 dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer;
4002 if (ll == IB_LINK_LAYER_ETHERNET)
4003 dev->ib_dev.get_netdev = mlx5_ib_get_netdev;
4004 dev->ib_dev.query_gid = mlx5_ib_query_gid;
4005 dev->ib_dev.add_gid = mlx5_ib_add_gid;
4006 dev->ib_dev.del_gid = mlx5_ib_del_gid;
4007 dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
4008 dev->ib_dev.modify_device = mlx5_ib_modify_device;
4009 dev->ib_dev.modify_port = mlx5_ib_modify_port;
4010 dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
4011 dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;
4012 dev->ib_dev.mmap = mlx5_ib_mmap;
4013 dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd;
4014 dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd;
4015 dev->ib_dev.create_ah = mlx5_ib_create_ah;
4016 dev->ib_dev.query_ah = mlx5_ib_query_ah;
4017 dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah;
4018 dev->ib_dev.create_srq = mlx5_ib_create_srq;
4019 dev->ib_dev.modify_srq = mlx5_ib_modify_srq;
4020 dev->ib_dev.query_srq = mlx5_ib_query_srq;
4021 dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq;
4022 dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv;
4023 dev->ib_dev.create_qp = mlx5_ib_create_qp;
4024 dev->ib_dev.modify_qp = mlx5_ib_modify_qp;
4025 dev->ib_dev.query_qp = mlx5_ib_query_qp;
4026 dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp;
4027 dev->ib_dev.post_send = mlx5_ib_post_send;
4028 dev->ib_dev.post_recv = mlx5_ib_post_recv;
4029 dev->ib_dev.create_cq = mlx5_ib_create_cq;
4030 dev->ib_dev.modify_cq = mlx5_ib_modify_cq;
4031 dev->ib_dev.resize_cq = mlx5_ib_resize_cq;
4032 dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq;
4033 dev->ib_dev.poll_cq = mlx5_ib_poll_cq;
4034 dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq;
4035 dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
4036 dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
4037 dev->ib_dev.rereg_user_mr = mlx5_ib_rereg_user_mr;
4038 dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
4039 dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
4040 dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
4041 dev->ib_dev.process_mad = mlx5_ib_process_mad;
4042 dev->ib_dev.alloc_mr = mlx5_ib_alloc_mr;
4043 dev->ib_dev.map_mr_sg = mlx5_ib_map_mr_sg;
4044 dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status;
4045 dev->ib_dev.get_port_immutable = mlx5_port_immutable;
4046 dev->ib_dev.get_dev_fw_str = get_dev_fw_str;
4047 dev->ib_dev.get_vector_affinity = mlx5_ib_get_vector_affinity;
4048 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads))
4049 dev->ib_dev.alloc_rdma_netdev = mlx5_ib_alloc_rdma_netdev;
4050
4051 if (mlx5_core_is_pf(mdev)) {
4052 dev->ib_dev.get_vf_config = mlx5_ib_get_vf_config;
4053 dev->ib_dev.set_vf_link_state = mlx5_ib_set_vf_link_state;
4054 dev->ib_dev.get_vf_stats = mlx5_ib_get_vf_stats;
4055 dev->ib_dev.set_vf_guid = mlx5_ib_set_vf_guid;
4056 }
4057
4058 dev->ib_dev.disassociate_ucontext = mlx5_ib_disassociate_ucontext;
4059
4060 mlx5_ib_internal_fill_odp_caps(dev);
4061
4062 dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4063
4064 if (MLX5_CAP_GEN(mdev, imaicl)) {
4065 dev->ib_dev.alloc_mw = mlx5_ib_alloc_mw;
4066 dev->ib_dev.dealloc_mw = mlx5_ib_dealloc_mw;
4067 dev->ib_dev.uverbs_cmd_mask |=
4068 (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
4069 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
4070 }
4071
4072 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
4073 dev->ib_dev.get_hw_stats = mlx5_ib_get_hw_stats;
4074 dev->ib_dev.alloc_hw_stats = mlx5_ib_alloc_hw_stats;
4075 }
4076
4077 if (MLX5_CAP_GEN(mdev, xrc)) {
4078 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
4079 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
4080 dev->ib_dev.uverbs_cmd_mask |=
4081 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
4082 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
4083 }
4084
4085 dev->ib_dev.create_flow = mlx5_ib_create_flow;
4086 dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
4087 dev->ib_dev.uverbs_ex_cmd_mask |=
4088 (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
4089 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
4090
4091 if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
4092 IB_LINK_LAYER_ETHERNET) {
4093 dev->ib_dev.create_wq = mlx5_ib_create_wq;
4094 dev->ib_dev.modify_wq = mlx5_ib_modify_wq;
4095 dev->ib_dev.destroy_wq = mlx5_ib_destroy_wq;
4096 dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table;
4097 dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table;
4098 dev->ib_dev.uverbs_ex_cmd_mask |=
4099 (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
4100 (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
4101 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
4102 (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
4103 (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
4104 }
4105 err = init_node_data(dev);
4106 if (err)
4107 goto err_free_port;
4108
4109 mutex_init(&dev->flow_db.lock);
4110 mutex_init(&dev->cap_mask_mutex);
4111 INIT_LIST_HEAD(&dev->qp_list);
4112 spin_lock_init(&dev->reset_flow_resource_lock);
4113
4114 if (ll == IB_LINK_LAYER_ETHERNET) {
4115 err = mlx5_enable_eth(dev);
4116 if (err)
4117 goto err_free_port;
4118 dev->roce.last_port_state = IB_PORT_DOWN;
4119 }
4120
4121 err = create_dev_resources(&dev->devr);
4122 if (err)
4123 goto err_disable_eth;
4124
4125 err = mlx5_ib_odp_init_one(dev);
4126 if (err)
4127 goto err_rsrc;
4128
4129 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
4130 err = mlx5_ib_alloc_counters(dev);
4131 if (err)
4132 goto err_odp;
4133 }
4134
4135 err = mlx5_ib_init_cong_debugfs(dev);
4136 if (err)
4137 goto err_cnt;
4138
4139 dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
4140 if (!dev->mdev->priv.uar)
4141 goto err_cong;
4142
4143 err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4144 if (err)
4145 goto err_uar_page;
4146
4147 err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4148 if (err)
4149 goto err_bfreg;
4150
4151 err = ib_register_device(&dev->ib_dev, NULL);
4152 if (err)
4153 goto err_fp_bfreg;
4154
4155 err = create_umr_res(dev);
4156 if (err)
4157 goto err_dev;
4158
4159 init_delay_drop(dev);
4160
4161 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
4162 err = device_create_file(&dev->ib_dev.dev,
4163 mlx5_class_attributes[i]);
4164 if (err)
4165 goto err_delay_drop;
4166 }
4167
4168 if ((MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
4169 (MLX5_CAP_GEN(mdev, disable_local_lb_uc) ||
4170 MLX5_CAP_GEN(mdev, disable_local_lb_mc)))
4171 mutex_init(&dev->lb_mutex);
4172
4173 dev->ib_active = true;
4174
4175 return dev;
4176
4177 err_delay_drop:
4178 cancel_delay_drop(dev);
4179 destroy_umrc_res(dev);
4180
4181 err_dev:
4182 ib_unregister_device(&dev->ib_dev);
4183
4184 err_fp_bfreg:
4185 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4186
4187 err_bfreg:
4188 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4189
4190 err_uar_page:
4191 mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
4192
4193 err_cong:
4194 mlx5_ib_cleanup_cong_debugfs(dev);
4195 err_cnt:
4196 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
4197 mlx5_ib_dealloc_counters(dev);
4198
4199 err_odp:
4200 mlx5_ib_odp_remove_one(dev);
4201
4202 err_rsrc:
4203 destroy_dev_resources(&dev->devr);
4204
4205 err_disable_eth:
4206 if (ll == IB_LINK_LAYER_ETHERNET) {
4207 mlx5_disable_eth(dev);
4208 mlx5_remove_netdev_notifier(dev);
4209 }
4210
4211 err_free_port:
4212 kfree(dev->port);
4213
4214 err_dealloc:
4215 ib_dealloc_device((struct ib_device *)dev);
4216
4217 return NULL;
4218 }
4219
4220 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
4221 {
4222 struct mlx5_ib_dev *dev = context;
4223 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
4224
4225 cancel_delay_drop(dev);
4226 mlx5_remove_netdev_notifier(dev);
4227 ib_unregister_device(&dev->ib_dev);
4228 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4229 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4230 mlx5_put_uars_page(dev->mdev, mdev->priv.uar);
4231 mlx5_ib_cleanup_cong_debugfs(dev);
4232 if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
4233 mlx5_ib_dealloc_counters(dev);
4234 destroy_umrc_res(dev);
4235 mlx5_ib_odp_remove_one(dev);
4236 destroy_dev_resources(&dev->devr);
4237 if (ll == IB_LINK_LAYER_ETHERNET)
4238 mlx5_disable_eth(dev);
4239 kfree(dev->port);
4240 ib_dealloc_device(&dev->ib_dev);
4241 }
4242
4243 static struct mlx5_interface mlx5_ib_interface = {
4244 .add = mlx5_ib_add,
4245 .remove = mlx5_ib_remove,
4246 .event = mlx5_ib_event,
4247 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
4248 .pfault = mlx5_ib_pfault,
4249 #endif
4250 .protocol = MLX5_INTERFACE_PROTOCOL_IB,
4251 };
4252
4253 static int __init mlx5_ib_init(void)
4254 {
4255 int err;
4256
4257 mlx5_ib_odp_init();
4258
4259 err = mlx5_register_interface(&mlx5_ib_interface);
4260
4261 return err;
4262 }
4263
4264 static void __exit mlx5_ib_cleanup(void)
4265 {
4266 mlx5_unregister_interface(&mlx5_ib_interface);
4267 }
4268
4269 module_init(mlx5_ib_init);
4270 module_exit(mlx5_ib_cleanup);