projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'hfi1' into k.o/for-4.14
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / infiniband / hw / hfi1 / vnic_main.c
1 /*
2 * Copyright(c) 2017 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48 /*
49 * This file contains HFI1 support for VNIC functionality
50 */
51
52 #include <linux/io.h>
53 #include <linux/if_vlan.h>
54
55 #include "vnic.h"
56
57 #define HFI_TX_TIMEOUT_MS 1000
58
59 #define HFI1_VNIC_RCV_Q_SIZE 1024
60
61 #define HFI1_VNIC_UP 0
62
63 static DEFINE_SPINLOCK(vport_cntr_lock);
64
65 static int setup_vnic_ctxt(struct hfi1_devdata *dd, struct hfi1_ctxtdata *uctxt)
66 {
67 unsigned int rcvctrl_ops = 0;
68 int ret;
69
70 hfi1_init_ctxt(uctxt->sc);
71
72 uctxt->do_interrupt = &handle_receive_interrupt;
73
74 /* Now allocate the RcvHdr queue and eager buffers. */
75 ret = hfi1_create_rcvhdrq(dd, uctxt);
76 if (ret)
77 goto done;
78
79 ret = hfi1_setup_eagerbufs(uctxt);
80 if (ret)
81 goto done;
82
83 if (uctxt->rcvhdrtail_kvaddr)
84 clear_rcvhdrtail(uctxt);
85
86 rcvctrl_ops = HFI1_RCVCTRL_CTXT_ENB;
87 rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_ENB;
88
89 if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
90 rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
91 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
92 rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
93 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
94 rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
95 if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
96 rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
97
98 hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt->ctxt);
99
100 uctxt->is_vnic = true;
101 done:
102 return ret;
103 }
104
105 static int allocate_vnic_ctxt(struct hfi1_devdata *dd,
106 struct hfi1_ctxtdata **vnic_ctxt)
107 {
108 struct hfi1_ctxtdata *uctxt;
109 unsigned int ctxt;
110 int ret;
111
112 if (dd->flags & HFI1_FROZEN)
113 return -EIO;
114
115 for (ctxt = dd->first_dyn_alloc_ctxt;
116 ctxt < dd->num_rcv_contexts; ctxt++)
117 if (!dd->rcd[ctxt])
118 break;
119
120 if (ctxt == dd->num_rcv_contexts)
121 return -EBUSY;
122
123 uctxt = hfi1_create_ctxtdata(dd->pport, ctxt, dd->node);
124 if (!uctxt) {
125 dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
126 return -ENOMEM;
127 }
128
129 uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
130 HFI1_CAP_KGET(NODROP_RHQ_FULL) |
131 HFI1_CAP_KGET(NODROP_EGR_FULL) |
132 HFI1_CAP_KGET(DMA_RTAIL);
133 uctxt->seq_cnt = 1;
134
135 /* Allocate and enable a PIO send context */
136 uctxt->sc = sc_alloc(dd, SC_VNIC, uctxt->rcvhdrqentsize,
137 uctxt->numa_id);
138
139 ret = uctxt->sc ? 0 : -ENOMEM;
140 if (ret)
141 goto bail;
142
143 dd_dev_dbg(dd, "allocated vnic send context %u(%u)\n",
144 uctxt->sc->sw_index, uctxt->sc->hw_context);
145 ret = sc_enable(uctxt->sc);
146 if (ret)
147 goto bail;
148
149 if (dd->num_msix_entries)
150 hfi1_set_vnic_msix_info(uctxt);
151
152 hfi1_stats.sps_ctxts++;
153 dd_dev_dbg(dd, "created vnic context %d\n", uctxt->ctxt);
154 *vnic_ctxt = uctxt;
155
156 return ret;
157 bail:
158 /*
159 * hfi1_rcd_put() will call hfi1_free_ctxtdata(), which will
160 * release send_context structure if uctxt->sc is not null
161 */
162 dd->rcd[uctxt->ctxt] = NULL;
163 hfi1_rcd_put(uctxt);
164 dd_dev_dbg(dd, "vnic allocation failed. rc %d\n", ret);
165 return ret;
166 }
167
168 static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,
169 struct hfi1_ctxtdata *uctxt)
170 {
171 unsigned long flags;
172
173 dd_dev_dbg(dd, "closing vnic context %d\n", uctxt->ctxt);
174 flush_wc();
175
176 if (dd->num_msix_entries)
177 hfi1_reset_vnic_msix_info(uctxt);
178
179 spin_lock_irqsave(&dd->uctxt_lock, flags);
180 /*
181 * Disable receive context and interrupt available, reset all
182 * RcvCtxtCtrl bits to default values.
183 */
184 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
185 HFI1_RCVCTRL_TIDFLOW_DIS |
186 HFI1_RCVCTRL_INTRAVAIL_DIS |
187 HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
188 HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
189 HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt->ctxt);
190 /*
191 * VNIC contexts are allocated from user context pool.
192 * Release them back to user context pool.
193 *
194 * Reset context integrity checks to default.
195 * (writes to CSRs probably belong in chip.c)
196 */
197 write_kctxt_csr(dd, uctxt->sc->hw_context, SEND_CTXT_CHECK_ENABLE,
198 hfi1_pkt_default_send_ctxt_mask(dd, SC_USER));
199 sc_disable(uctxt->sc);
200
201 dd->send_contexts[uctxt->sc->sw_index].type = SC_USER;
202 spin_unlock_irqrestore(&dd->uctxt_lock, flags);
203
204 dd->rcd[uctxt->ctxt] = NULL;
205 uctxt->event_flags = 0;
206
207 hfi1_clear_tids(uctxt);
208 hfi1_clear_ctxt_pkey(dd, uctxt);
209
210 hfi1_stats.sps_ctxts--;
211 hfi1_rcd_put(uctxt);
212 }
213
214 void hfi1_vnic_setup(struct hfi1_devdata *dd)
215 {
216 idr_init(&dd->vnic.vesw_idr);
217 }
218
219 void hfi1_vnic_cleanup(struct hfi1_devdata *dd)
220 {
221 idr_destroy(&dd->vnic.vesw_idr);
222 }
223
224 #define SUM_GRP_COUNTERS(stats, qstats, x_grp) do { \
225 u64 *src64, *dst64; \
226 for (src64 = &qstats->x_grp.unicast, \
227 dst64 = &stats->x_grp.unicast; \
228 dst64 <= &stats->x_grp.s_1519_max;) { \
229 *dst64++ += *src64++; \
230 } \
231 } while (0)
232
233 /* hfi1_vnic_update_stats - update statistics */
234 static void hfi1_vnic_update_stats(struct hfi1_vnic_vport_info *vinfo,
235 struct opa_vnic_stats *stats)
236 {
237 struct net_device *netdev = vinfo->netdev;
238 u8 i;
239
240 /* add tx counters on different queues */
241 for (i = 0; i < vinfo->num_tx_q; i++) {
242 struct opa_vnic_stats *qstats = &vinfo->stats[i];
243 struct rtnl_link_stats64 *qnstats = &vinfo->stats[i].netstats;
244
245 stats->netstats.tx_fifo_errors += qnstats->tx_fifo_errors;
246 stats->netstats.tx_carrier_errors += qnstats->tx_carrier_errors;
247 stats->tx_drop_state += qstats->tx_drop_state;
248 stats->tx_dlid_zero += qstats->tx_dlid_zero;
249
250 SUM_GRP_COUNTERS(stats, qstats, tx_grp);
251 stats->netstats.tx_packets += qnstats->tx_packets;
252 stats->netstats.tx_bytes += qnstats->tx_bytes;
253 }
254
255 /* add rx counters on different queues */
256 for (i = 0; i < vinfo->num_rx_q; i++) {
257 struct opa_vnic_stats *qstats = &vinfo->stats[i];
258 struct rtnl_link_stats64 *qnstats = &vinfo->stats[i].netstats;
259
260 stats->netstats.rx_fifo_errors += qnstats->rx_fifo_errors;
261 stats->netstats.rx_nohandler += qnstats->rx_nohandler;
262 stats->rx_drop_state += qstats->rx_drop_state;
263 stats->rx_oversize += qstats->rx_oversize;
264 stats->rx_runt += qstats->rx_runt;
265
266 SUM_GRP_COUNTERS(stats, qstats, rx_grp);
267 stats->netstats.rx_packets += qnstats->rx_packets;
268 stats->netstats.rx_bytes += qnstats->rx_bytes;
269 }
270
271 stats->netstats.tx_errors = stats->netstats.tx_fifo_errors +
272 stats->netstats.tx_carrier_errors +
273 stats->tx_drop_state + stats->tx_dlid_zero;
274 stats->netstats.tx_dropped = stats->netstats.tx_errors;
275
276 stats->netstats.rx_errors = stats->netstats.rx_fifo_errors +
277 stats->netstats.rx_nohandler +
278 stats->rx_drop_state + stats->rx_oversize +
279 stats->rx_runt;
280 stats->netstats.rx_dropped = stats->netstats.rx_errors;
281
282 netdev->stats.tx_packets = stats->netstats.tx_packets;
283 netdev->stats.tx_bytes = stats->netstats.tx_bytes;
284 netdev->stats.tx_fifo_errors = stats->netstats.tx_fifo_errors;
285 netdev->stats.tx_carrier_errors = stats->netstats.tx_carrier_errors;
286 netdev->stats.tx_errors = stats->netstats.tx_errors;
287 netdev->stats.tx_dropped = stats->netstats.tx_dropped;
288
289 netdev->stats.rx_packets = stats->netstats.rx_packets;
290 netdev->stats.rx_bytes = stats->netstats.rx_bytes;
291 netdev->stats.rx_fifo_errors = stats->netstats.rx_fifo_errors;
292 netdev->stats.multicast = stats->rx_grp.mcastbcast;
293 netdev->stats.rx_length_errors = stats->rx_oversize + stats->rx_runt;
294 netdev->stats.rx_errors = stats->netstats.rx_errors;
295 netdev->stats.rx_dropped = stats->netstats.rx_dropped;
296 }
297
298 /* update_len_counters - update pkt's len histogram counters */
299 static inline void update_len_counters(struct opa_vnic_grp_stats *grp,
300 int len)
301 {
302 /* account for 4 byte FCS */
303 if (len >= 1515)
304 grp->s_1519_max++;
305 else if (len >= 1020)
306 grp->s_1024_1518++;
307 else if (len >= 508)
308 grp->s_512_1023++;
309 else if (len >= 252)
310 grp->s_256_511++;
311 else if (len >= 124)
312 grp->s_128_255++;
313 else if (len >= 61)
314 grp->s_65_127++;
315 else
316 grp->s_64++;
317 }
318
319 /* hfi1_vnic_update_tx_counters - update transmit counters */
320 static void hfi1_vnic_update_tx_counters(struct hfi1_vnic_vport_info *vinfo,
321 u8 q_idx, struct sk_buff *skb, int err)
322 {
323 struct ethhdr *mac_hdr = (struct ethhdr *)skb_mac_header(skb);
324 struct opa_vnic_stats *stats = &vinfo->stats[q_idx];
325 struct opa_vnic_grp_stats *tx_grp = &stats->tx_grp;
326 u16 vlan_tci;
327
328 stats->netstats.tx_packets++;
329 stats->netstats.tx_bytes += skb->len + ETH_FCS_LEN;
330
331 update_len_counters(tx_grp, skb->len);
332
333 /* rest of the counts are for good packets only */
334 if (unlikely(err))
335 return;
336
337 if (is_multicast_ether_addr(mac_hdr->h_dest))
338 tx_grp->mcastbcast++;
339 else
340 tx_grp->unicast++;
341
342 if (!__vlan_get_tag(skb, &vlan_tci))
343 tx_grp->vlan++;
344 else
345 tx_grp->untagged++;
346 }
347
348 /* hfi1_vnic_update_rx_counters - update receive counters */
349 static void hfi1_vnic_update_rx_counters(struct hfi1_vnic_vport_info *vinfo,
350 u8 q_idx, struct sk_buff *skb, int err)
351 {
352 struct ethhdr *mac_hdr = (struct ethhdr *)skb->data;
353 struct opa_vnic_stats *stats = &vinfo->stats[q_idx];
354 struct opa_vnic_grp_stats *rx_grp = &stats->rx_grp;
355 u16 vlan_tci;
356
357 stats->netstats.rx_packets++;
358 stats->netstats.rx_bytes += skb->len + ETH_FCS_LEN;
359
360 update_len_counters(rx_grp, skb->len);
361
362 /* rest of the counts are for good packets only */
363 if (unlikely(err))
364 return;
365
366 if (is_multicast_ether_addr(mac_hdr->h_dest))
367 rx_grp->mcastbcast++;
368 else
369 rx_grp->unicast++;
370
371 if (!__vlan_get_tag(skb, &vlan_tci))
372 rx_grp->vlan++;
373 else
374 rx_grp->untagged++;
375 }
376
377 /* This function is overloaded for opa_vnic specific implementation */
378 static void hfi1_vnic_get_stats64(struct net_device *netdev,
379 struct rtnl_link_stats64 *stats)
380 {
381 struct opa_vnic_stats *vstats = (struct opa_vnic_stats *)stats;
382 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
383
384 hfi1_vnic_update_stats(vinfo, vstats);
385 }
386
387 static u64 create_bypass_pbc(u32 vl, u32 dw_len)
388 {
389 u64 pbc;
390
391 pbc = ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
392 | PBC_INSERT_BYPASS_ICRC | PBC_CREDIT_RETURN
393 | PBC_PACKET_BYPASS
394 | ((vl & PBC_VL_MASK) << PBC_VL_SHIFT)
395 | (dw_len & PBC_LENGTH_DWS_MASK) << PBC_LENGTH_DWS_SHIFT;
396
397 return pbc;
398 }
399
400 /* hfi1_vnic_maybe_stop_tx - stop tx queue if required */
401 static void hfi1_vnic_maybe_stop_tx(struct hfi1_vnic_vport_info *vinfo,
402 u8 q_idx)
403 {
404 netif_stop_subqueue(vinfo->netdev, q_idx);
405 if (!hfi1_vnic_sdma_write_avail(vinfo, q_idx))
406 return;
407
408 netif_start_subqueue(vinfo->netdev, q_idx);
409 }
410
411 static netdev_tx_t hfi1_netdev_start_xmit(struct sk_buff *skb,
412 struct net_device *netdev)
413 {
414 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
415 u8 pad_len, q_idx = skb->queue_mapping;
416 struct hfi1_devdata *dd = vinfo->dd;
417 struct opa_vnic_skb_mdata *mdata;
418 u32 pkt_len, total_len;
419 int err = -EINVAL;
420 u64 pbc;
421
422 v_dbg("xmit: queue %d skb len %d\n", q_idx, skb->len);
423 if (unlikely(!netif_oper_up(netdev))) {
424 vinfo->stats[q_idx].tx_drop_state++;
425 goto tx_finish;
426 }
427
428 /* take out meta data */
429 mdata = (struct opa_vnic_skb_mdata *)skb->data;
430 skb_pull(skb, sizeof(*mdata));
431 if (unlikely(mdata->flags & OPA_VNIC_SKB_MDATA_ENCAP_ERR)) {
432 vinfo->stats[q_idx].tx_dlid_zero++;
433 goto tx_finish;
434 }
435
436 /* add tail padding (for 8 bytes size alignment) and icrc */
437 pad_len = -(skb->len + OPA_VNIC_ICRC_TAIL_LEN) & 0x7;
438 pad_len += OPA_VNIC_ICRC_TAIL_LEN;
439
440 /*
441 * pkt_len is how much data we have to write, includes header and data.
442 * total_len is length of the packet in Dwords plus the PBC should not
443 * include the CRC.
444 */
445 pkt_len = (skb->len + pad_len) >> 2;
446 total_len = pkt_len + 2; /* PBC + packet */
447
448 pbc = create_bypass_pbc(mdata->vl, total_len);
449
450 skb_get(skb);
451 v_dbg("pbc 0x%016llX len %d pad_len %d\n", pbc, skb->len, pad_len);
452 err = dd->process_vnic_dma_send(dd, q_idx, vinfo, skb, pbc, pad_len);
453 if (unlikely(err)) {
454 if (err == -ENOMEM)
455 vinfo->stats[q_idx].netstats.tx_fifo_errors++;
456 else if (err != -EBUSY)
457 vinfo->stats[q_idx].netstats.tx_carrier_errors++;
458 }
459 /* remove the header before updating tx counters */
460 skb_pull(skb, OPA_VNIC_HDR_LEN);
461
462 if (unlikely(err == -EBUSY)) {
463 hfi1_vnic_maybe_stop_tx(vinfo, q_idx);
464 dev_kfree_skb_any(skb);
465 return NETDEV_TX_BUSY;
466 }
467
468 tx_finish:
469 /* update tx counters */
470 hfi1_vnic_update_tx_counters(vinfo, q_idx, skb, err);
471 dev_kfree_skb_any(skb);
472 return NETDEV_TX_OK;
473 }
474
475 static u16 hfi1_vnic_select_queue(struct net_device *netdev,
476 struct sk_buff *skb,
477 void *accel_priv,
478 select_queue_fallback_t fallback)
479 {
480 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
481 struct opa_vnic_skb_mdata *mdata;
482 struct sdma_engine *sde;
483
484 mdata = (struct opa_vnic_skb_mdata *)skb->data;
485 sde = sdma_select_engine_vl(vinfo->dd, mdata->entropy, mdata->vl);
486 return sde->this_idx;
487 }
488
489 /* hfi1_vnic_decap_skb - strip OPA header from the skb (ethernet) packet */
490 static inline int hfi1_vnic_decap_skb(struct hfi1_vnic_rx_queue *rxq,
491 struct sk_buff *skb)
492 {
493 struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
494 int max_len = vinfo->netdev->mtu + VLAN_ETH_HLEN;
495 int rc = -EFAULT;
496
497 skb_pull(skb, OPA_VNIC_HDR_LEN);
498
499 /* Validate Packet length */
500 if (unlikely(skb->len > max_len))
501 vinfo->stats[rxq->idx].rx_oversize++;
502 else if (unlikely(skb->len < ETH_ZLEN))
503 vinfo->stats[rxq->idx].rx_runt++;
504 else
505 rc = 0;
506 return rc;
507 }
508
509 static inline struct sk_buff *hfi1_vnic_get_skb(struct hfi1_vnic_rx_queue *rxq)
510 {
511 unsigned char *pad_info;
512 struct sk_buff *skb;
513
514 skb = skb_dequeue(&rxq->skbq);
515 if (unlikely(!skb))
516 return NULL;
517
518 /* remove tail padding and icrc */
519 pad_info = skb->data + skb->len - 1;
520 skb_trim(skb, (skb->len - OPA_VNIC_ICRC_TAIL_LEN -
521 ((*pad_info) & 0x7)));
522
523 return skb;
524 }
525
526 /* hfi1_vnic_handle_rx - handle skb receive */
527 static void hfi1_vnic_handle_rx(struct hfi1_vnic_rx_queue *rxq,
528 int *work_done, int work_to_do)
529 {
530 struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
531 struct sk_buff *skb;
532 int rc;
533
534 while (1) {
535 if (*work_done >= work_to_do)
536 break;
537
538 skb = hfi1_vnic_get_skb(rxq);
539 if (unlikely(!skb))
540 break;
541
542 rc = hfi1_vnic_decap_skb(rxq, skb);
543 /* update rx counters */
544 hfi1_vnic_update_rx_counters(vinfo, rxq->idx, skb, rc);
545 if (unlikely(rc)) {
546 dev_kfree_skb_any(skb);
547 continue;
548 }
549
550 skb_checksum_none_assert(skb);
551 skb->protocol = eth_type_trans(skb, rxq->netdev);
552
553 napi_gro_receive(&rxq->napi, skb);
554 (*work_done)++;
555 }
556 }
557
558 /* hfi1_vnic_napi - napi receive polling callback function */
559 static int hfi1_vnic_napi(struct napi_struct *napi, int budget)
560 {
561 struct hfi1_vnic_rx_queue *rxq = container_of(napi,
562 struct hfi1_vnic_rx_queue, napi);
563 struct hfi1_vnic_vport_info *vinfo = rxq->vinfo;
564 int work_done = 0;
565
566 v_dbg("napi %d budget %d\n", rxq->idx, budget);
567 hfi1_vnic_handle_rx(rxq, &work_done, budget);
568
569 v_dbg("napi %d work_done %d\n", rxq->idx, work_done);
570 if (work_done < budget)
571 napi_complete(napi);
572
573 return work_done;
574 }
575
576 void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
577 {
578 struct hfi1_devdata *dd = packet->rcd->dd;
579 struct hfi1_vnic_vport_info *vinfo = NULL;
580 struct hfi1_vnic_rx_queue *rxq;
581 struct sk_buff *skb;
582 int l4_type, vesw_id = -1;
583 u8 q_idx;
584
585 l4_type = HFI1_GET_L4_TYPE(packet->ebuf);
586 if (likely(l4_type == OPA_VNIC_L4_ETHR)) {
587 vesw_id = HFI1_VNIC_GET_VESWID(packet->ebuf);
588 vinfo = idr_find(&dd->vnic.vesw_idr, vesw_id);
589
590 /*
591 * In case of invalid vesw id, count the error on
592 * the first available vport.
593 */
594 if (unlikely(!vinfo)) {
595 struct hfi1_vnic_vport_info *vinfo_tmp;
596 int id_tmp = 0;
597
598 vinfo_tmp = idr_get_next(&dd->vnic.vesw_idr, &id_tmp);
599 if (vinfo_tmp) {
600 spin_lock(&vport_cntr_lock);
601 vinfo_tmp->stats[0].netstats.rx_nohandler++;
602 spin_unlock(&vport_cntr_lock);
603 }
604 }
605 }
606
607 if (unlikely(!vinfo)) {
608 dd_dev_warn(dd, "vnic rcv err: l4 %d vesw id %d ctx %d\n",
609 l4_type, vesw_id, packet->rcd->ctxt);
610 return;
611 }
612
613 q_idx = packet->rcd->vnic_q_idx;
614 rxq = &vinfo->rxq[q_idx];
615 if (unlikely(!netif_oper_up(vinfo->netdev))) {
616 vinfo->stats[q_idx].rx_drop_state++;
617 skb_queue_purge(&rxq->skbq);
618 return;
619 }
620
621 if (unlikely(skb_queue_len(&rxq->skbq) > HFI1_VNIC_RCV_Q_SIZE)) {
622 vinfo->stats[q_idx].netstats.rx_fifo_errors++;
623 return;
624 }
625
626 skb = netdev_alloc_skb(vinfo->netdev, packet->tlen);
627 if (unlikely(!skb)) {
628 vinfo->stats[q_idx].netstats.rx_fifo_errors++;
629 return;
630 }
631
632 memcpy(skb->data, packet->ebuf, packet->tlen);
633 skb_put(skb, packet->tlen);
634 skb_queue_tail(&rxq->skbq, skb);
635
636 if (napi_schedule_prep(&rxq->napi)) {
637 v_dbg("napi %d scheduling\n", q_idx);
638 __napi_schedule(&rxq->napi);
639 }
640 }
641
642 static int hfi1_vnic_up(struct hfi1_vnic_vport_info *vinfo)
643 {
644 struct hfi1_devdata *dd = vinfo->dd;
645 struct net_device *netdev = vinfo->netdev;
646 int i, rc;
647
648 /* ensure virtual eth switch id is valid */
649 if (!vinfo->vesw_id)
650 return -EINVAL;
651
652 rc = idr_alloc(&dd->vnic.vesw_idr, vinfo, vinfo->vesw_id,
653 vinfo->vesw_id + 1, GFP_NOWAIT);
654 if (rc < 0)
655 return rc;
656
657 for (i = 0; i < vinfo->num_rx_q; i++) {
658 struct hfi1_vnic_rx_queue *rxq = &vinfo->rxq[i];
659
660 skb_queue_head_init(&rxq->skbq);
661 napi_enable(&rxq->napi);
662 }
663
664 netif_carrier_on(netdev);
665 netif_tx_start_all_queues(netdev);
666 set_bit(HFI1_VNIC_UP, &vinfo->flags);
667
668 return 0;
669 }
670
671 static void hfi1_vnic_down(struct hfi1_vnic_vport_info *vinfo)
672 {
673 struct hfi1_devdata *dd = vinfo->dd;
674 u8 i;
675
676 clear_bit(HFI1_VNIC_UP, &vinfo->flags);
677 netif_carrier_off(vinfo->netdev);
678 netif_tx_disable(vinfo->netdev);
679 idr_remove(&dd->vnic.vesw_idr, vinfo->vesw_id);
680
681 /* ensure irqs see the change */
682 hfi1_vnic_synchronize_irq(dd);
683
684 /* remove unread skbs */
685 for (i = 0; i < vinfo->num_rx_q; i++) {
686 struct hfi1_vnic_rx_queue *rxq = &vinfo->rxq[i];
687
688 napi_disable(&rxq->napi);
689 skb_queue_purge(&rxq->skbq);
690 }
691 }
692
693 static int hfi1_netdev_open(struct net_device *netdev)
694 {
695 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
696 int rc;
697
698 mutex_lock(&vinfo->lock);
699 rc = hfi1_vnic_up(vinfo);
700 mutex_unlock(&vinfo->lock);
701 return rc;
702 }
703
704 static int hfi1_netdev_close(struct net_device *netdev)
705 {
706 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
707
708 mutex_lock(&vinfo->lock);
709 if (test_bit(HFI1_VNIC_UP, &vinfo->flags))
710 hfi1_vnic_down(vinfo);
711 mutex_unlock(&vinfo->lock);
712 return 0;
713 }
714
715 static int hfi1_vnic_allot_ctxt(struct hfi1_devdata *dd,
716 struct hfi1_ctxtdata **vnic_ctxt)
717 {
718 int rc;
719
720 rc = allocate_vnic_ctxt(dd, vnic_ctxt);
721 if (rc) {
722 dd_dev_err(dd, "vnic ctxt alloc failed %d\n", rc);
723 return rc;
724 }
725
726 rc = setup_vnic_ctxt(dd, *vnic_ctxt);
727 if (rc) {
728 dd_dev_err(dd, "vnic ctxt setup failed %d\n", rc);
729 deallocate_vnic_ctxt(dd, *vnic_ctxt);
730 *vnic_ctxt = NULL;
731 }
732
733 return rc;
734 }
735
736 static int hfi1_vnic_init(struct hfi1_vnic_vport_info *vinfo)
737 {
738 struct hfi1_devdata *dd = vinfo->dd;
739 int i, rc = 0;
740
741 mutex_lock(&hfi1_mutex);
742 if (!dd->vnic.num_vports) {
743 rc = hfi1_vnic_txreq_init(dd);
744 if (rc)
745 goto txreq_fail;
746
747 dd->vnic.msix_idx = dd->first_dyn_msix_idx;
748 }
749
750 for (i = dd->vnic.num_ctxt; i < vinfo->num_rx_q; i++) {
751 rc = hfi1_vnic_allot_ctxt(dd, &dd->vnic.ctxt[i]);
752 if (rc)
753 break;
754 hfi1_rcd_get(dd->vnic.ctxt[i]);
755 dd->vnic.ctxt[i]->vnic_q_idx = i;
756 }
757
758 if (i < vinfo->num_rx_q) {
759 /*
760 * If required amount of contexts is not
761 * allocated successfully then remaining contexts
762 * are released.
763 */
764 while (i-- > dd->vnic.num_ctxt) {
765 deallocate_vnic_ctxt(dd, dd->vnic.ctxt[i]);
766 hfi1_rcd_put(dd->vnic.ctxt[i]);
767 dd->vnic.ctxt[i] = NULL;
768 }
769 goto alloc_fail;
770 }
771
772 if (dd->vnic.num_ctxt != i) {
773 dd->vnic.num_ctxt = i;
774 hfi1_init_vnic_rsm(dd);
775 }
776
777 dd->vnic.num_vports++;
778 hfi1_vnic_sdma_init(vinfo);
779 alloc_fail:
780 if (!dd->vnic.num_vports)
781 hfi1_vnic_txreq_deinit(dd);
782 txreq_fail:
783 mutex_unlock(&hfi1_mutex);
784 return rc;
785 }
786
787 static void hfi1_vnic_deinit(struct hfi1_vnic_vport_info *vinfo)
788 {
789 struct hfi1_devdata *dd = vinfo->dd;
790 int i;
791
792 mutex_lock(&hfi1_mutex);
793 if (--dd->vnic.num_vports == 0) {
794 for (i = 0; i < dd->vnic.num_ctxt; i++) {
795 deallocate_vnic_ctxt(dd, dd->vnic.ctxt[i]);
796 hfi1_rcd_put(dd->vnic.ctxt[i]);
797 dd->vnic.ctxt[i] = NULL;
798 }
799 hfi1_deinit_vnic_rsm(dd);
800 dd->vnic.num_ctxt = 0;
801 hfi1_vnic_txreq_deinit(dd);
802 }
803 mutex_unlock(&hfi1_mutex);
804 }
805
806 static void hfi1_vnic_set_vesw_id(struct net_device *netdev, int id)
807 {
808 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
809 bool reopen = false;
810
811 /*
812 * If vesw_id is being changed, and if the vnic port is up,
813 * reset the vnic port to ensure new vesw_id gets picked up
814 */
815 if (id != vinfo->vesw_id) {
816 mutex_lock(&vinfo->lock);
817 if (test_bit(HFI1_VNIC_UP, &vinfo->flags)) {
818 hfi1_vnic_down(vinfo);
819 reopen = true;
820 }
821
822 vinfo->vesw_id = id;
823 if (reopen)
824 hfi1_vnic_up(vinfo);
825
826 mutex_unlock(&vinfo->lock);
827 }
828 }
829
830 /* netdev ops */
831 static const struct net_device_ops hfi1_netdev_ops = {
832 .ndo_open = hfi1_netdev_open,
833 .ndo_stop = hfi1_netdev_close,
834 .ndo_start_xmit = hfi1_netdev_start_xmit,
835 .ndo_select_queue = hfi1_vnic_select_queue,
836 .ndo_get_stats64 = hfi1_vnic_get_stats64,
837 };
838
839 static void hfi1_vnic_free_rn(struct net_device *netdev)
840 {
841 struct hfi1_vnic_vport_info *vinfo = opa_vnic_dev_priv(netdev);
842
843 hfi1_vnic_deinit(vinfo);
844 mutex_destroy(&vinfo->lock);
845 free_netdev(netdev);
846 }
847
848 struct net_device *hfi1_vnic_alloc_rn(struct ib_device *device,
849 u8 port_num,
850 enum rdma_netdev_t type,
851 const char *name,
852 unsigned char name_assign_type,
853 void (*setup)(struct net_device *))
854 {
855 struct hfi1_devdata *dd = dd_from_ibdev(device);
856 struct hfi1_vnic_vport_info *vinfo;
857 struct net_device *netdev;
858 struct rdma_netdev *rn;
859 int i, size, rc;
860
861 if (!port_num || (port_num > dd->num_pports))
862 return ERR_PTR(-EINVAL);
863
864 if (type != RDMA_NETDEV_OPA_VNIC)
865 return ERR_PTR(-EOPNOTSUPP);
866
867 size = sizeof(struct opa_vnic_rdma_netdev) + sizeof(*vinfo);
868 netdev = alloc_netdev_mqs(size, name, name_assign_type, setup,
869 dd->chip_sdma_engines, HFI1_NUM_VNIC_CTXT);
870 if (!netdev)
871 return ERR_PTR(-ENOMEM);
872
873 rn = netdev_priv(netdev);
874 vinfo = opa_vnic_dev_priv(netdev);
875 vinfo->dd = dd;
876 vinfo->num_tx_q = dd->chip_sdma_engines;
877 vinfo->num_rx_q = HFI1_NUM_VNIC_CTXT;
878 vinfo->netdev = netdev;
879 rn->free_rdma_netdev = hfi1_vnic_free_rn;
880 rn->set_id = hfi1_vnic_set_vesw_id;
881
882 netdev->features = NETIF_F_HIGHDMA | NETIF_F_SG;
883 netdev->hw_features = netdev->features;
884 netdev->vlan_features = netdev->features;
885 netdev->watchdog_timeo = msecs_to_jiffies(HFI_TX_TIMEOUT_MS);
886 netdev->netdev_ops = &hfi1_netdev_ops;
887 mutex_init(&vinfo->lock);
888
889 for (i = 0; i < vinfo->num_rx_q; i++) {
890 struct hfi1_vnic_rx_queue *rxq = &vinfo->rxq[i];
891
892 rxq->idx = i;
893 rxq->vinfo = vinfo;
894 rxq->netdev = netdev;
895 netif_napi_add(netdev, &rxq->napi, hfi1_vnic_napi, 64);
896 }
897
898 rc = hfi1_vnic_init(vinfo);
899 if (rc)
900 goto init_fail;
901
902 return netdev;
903 init_fail:
904 mutex_destroy(&vinfo->lock);
905 free_netdev(netdev);
906 return ERR_PTR(rc);
907 }