From cb6aeb07929453d5ae127b536b14f6bd3d4c5942 Mon Sep 17 00:00:00 2001 From: "Mintz, Yuval" Date: Tue, 29 Nov 2016 16:47:10 +0200 Subject: [PATCH] qede: Add support for XDP_TX Add support for forwarding via XDP. Once the eBPF is attached, driver would allocate & configure a designated transmission queue meant solely for forwarding packets. Said queue would share the receive-queue's interrupt line, and would have it's own Tx statistics. Infrastructure changes required for this [spread-out through the code]: - Determine the DMA direction of the receive buffers based on the presence of the eBPF program. - Turn the sw Tx ring into a union, as regular/XDP queues have different needs for releasing resources after completion [regular requires the SKB, XDP requires the transmitted page]. Signed-off-by: Yuval Mintz Signed-off-by: David S. Miller --- drivers/net/ethernet/qlogic/qede/qede.h | 16 +- .../net/ethernet/qlogic/qede/qede_ethtool.c | 22 +- drivers/net/ethernet/qlogic/qede/qede_main.c | 213 +++++++++++++++--- 3 files changed, 216 insertions(+), 35 deletions(-) diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index cc14aed5f16e..c79dc78746fc 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -258,6 +258,7 @@ struct qede_rx_queue { u16 sw_rx_prod; u16 num_rx_buffers; /* Slowpath */ + u8 data_direction; u8 rxq_id; u32 rx_buf_size; @@ -294,6 +295,7 @@ struct sw_tx_bd { }; struct qede_tx_queue { + u8 is_xdp; bool is_legacy; u16 sw_tx_cons; u16 sw_tx_prod; @@ -310,8 +312,18 @@ struct qede_tx_queue { void __iomem *doorbell_addr; union db_prod tx_db; int index; /* Slowpath only */ +#define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \ + QEDE_MAX_TSS_CNT(edev)) +#define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev)) + + /* Regular Tx requires skb + metadata for release purpose, + * while XDP requires only the pages themselves. + */ + union { + struct sw_tx_bd *skbs; + struct page **pages; + } sw_tx_ring; - struct sw_tx_bd *sw_tx_ring; struct qed_chain tx_pbl; /* Slowpath; Should be kept in end [unless missing padding] */ @@ -336,10 +348,12 @@ struct qede_fastpath { #define QEDE_FASTPATH_COMBINED (QEDE_FASTPATH_TX | QEDE_FASTPATH_RX) u8 type; u8 id; + u8 xdp_xmit; struct napi_struct napi; struct qed_sb_info *sb_info; struct qede_rx_queue *rxq; struct qede_tx_queue *txq; + struct qede_tx_queue *xdp_tx; #define VEC_NAME_SIZE (sizeof(((struct net_device *)0)->name) + 8) char name[VEC_NAME_SIZE]; diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 6c70e298ba77..1c48f445c93b 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -165,8 +165,13 @@ static void qede_get_strings_stats_txq(struct qede_dev *edev, int i; for (i = 0; i < QEDE_NUM_TQSTATS; i++) { - sprintf(*buf, "%d: %s", txq->index, - qede_tqstats_arr[i].string); + if (txq->is_xdp) + sprintf(*buf, "%d [XDP]: %s", + QEDE_TXQ_XDP_TO_IDX(edev, txq), + qede_tqstats_arr[i].string); + else + sprintf(*buf, "%d: %s", txq->index, + qede_tqstats_arr[i].string); *buf += ETH_GSTRING_LEN; } } @@ -195,6 +200,9 @@ static void qede_get_strings_stats(struct qede_dev *edev, u8 *buf) if (fp->type & QEDE_FASTPATH_RX) qede_get_strings_stats_rxq(edev, fp->rxq, &buf); + if (fp->type & QEDE_FASTPATH_XDP) + qede_get_strings_stats_txq(edev, fp->xdp_tx, &buf); + if (fp->type & QEDE_FASTPATH_TX) qede_get_strings_stats_txq(edev, fp->txq, &buf); } @@ -268,6 +276,9 @@ static void qede_get_ethtool_stats(struct net_device *dev, if (fp->type & QEDE_FASTPATH_RX) qede_get_ethtool_stats_rxq(fp->rxq, &buf); + if (fp->type & QEDE_FASTPATH_XDP) + qede_get_ethtool_stats_txq(fp->xdp_tx, &buf); + if (fp->type & QEDE_FASTPATH_TX) qede_get_ethtool_stats_txq(fp->txq, &buf); } @@ -305,6 +316,9 @@ static int qede_get_sset_count(struct net_device *dev, int stringset) /* Account for the Regular Rx statistics */ num_stats += QEDE_RSS_COUNT(edev) * QEDE_NUM_RQSTATS; + /* Account for XDP statistics [if needed] */ + if (edev->xdp_prog) + num_stats += QEDE_RSS_COUNT(edev) * QEDE_NUM_TQSTATS; return num_stats; case ETH_SS_PRIV_FLAGS: @@ -1216,7 +1230,7 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev, /* Fill the entry in the SW ring and the BDs in the FW ring */ idx = txq->sw_tx_prod & NUM_TX_BDS_MAX; - txq->sw_tx_ring[idx].skb = skb; + txq->sw_tx_ring.skbs[idx].skb = skb; first_bd = qed_chain_produce(&txq->tx_pbl); memset(first_bd, 0, sizeof(*first_bd)); val = 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; @@ -1270,7 +1284,7 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev, dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), BD_UNMAP_LEN(first_bd), DMA_TO_DEVICE); txq->sw_tx_cons++; - txq->sw_tx_ring[idx].skb = NULL; + txq->sw_tx_ring.skbs[idx].skb = NULL; return 0; } diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 71c68c25d01a..172ff6da92ad 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -94,6 +94,9 @@ static int qede_probe(struct pci_dev *pdev, const struct pci_device_id *id); #define TX_TIMEOUT (5 * HZ) +/* Utilize last protocol index for XDP */ +#define XDP_PI 11 + static void qede_remove(struct pci_dev *pdev); static void qede_shutdown(struct pci_dev *pdev); static void qede_link_update(void *dev, struct qed_link_output *link); @@ -301,12 +304,12 @@ static int qede_free_tx_pkt(struct qede_dev *edev, struct qede_tx_queue *txq, int *len) { u16 idx = txq->sw_tx_cons & NUM_TX_BDS_MAX; - struct sk_buff *skb = txq->sw_tx_ring[idx].skb; + struct sk_buff *skb = txq->sw_tx_ring.skbs[idx].skb; struct eth_tx_1st_bd *first_bd; struct eth_tx_bd *tx_data_bd; int bds_consumed = 0; int nbds; - bool data_split = txq->sw_tx_ring[idx].flags & QEDE_TSO_SPLIT_BD; + bool data_split = txq->sw_tx_ring.skbs[idx].flags & QEDE_TSO_SPLIT_BD; int i, split_bd_len = 0; if (unlikely(!skb)) { @@ -346,8 +349,8 @@ static int qede_free_tx_pkt(struct qede_dev *edev, /* Free skb */ dev_kfree_skb_any(skb); - txq->sw_tx_ring[idx].skb = NULL; - txq->sw_tx_ring[idx].flags = 0; + txq->sw_tx_ring.skbs[idx].skb = NULL; + txq->sw_tx_ring.skbs[idx].flags = 0; return 0; } @@ -358,7 +361,7 @@ static void qede_free_failed_tx_pkt(struct qede_tx_queue *txq, int nbd, bool data_split) { u16 idx = txq->sw_tx_prod & NUM_TX_BDS_MAX; - struct sk_buff *skb = txq->sw_tx_ring[idx].skb; + struct sk_buff *skb = txq->sw_tx_ring.skbs[idx].skb; struct eth_tx_bd *tx_data_bd; int i, split_bd_len = 0; @@ -394,8 +397,8 @@ static void qede_free_failed_tx_pkt(struct qede_tx_queue *txq, /* Free skb */ dev_kfree_skb_any(skb); - txq->sw_tx_ring[idx].skb = NULL; - txq->sw_tx_ring[idx].flags = 0; + txq->sw_tx_ring.skbs[idx].skb = NULL; + txq->sw_tx_ring.skbs[idx].flags = 0; } static u32 qede_xmit_type(struct sk_buff *skb, int *ipv6_ext) @@ -530,6 +533,47 @@ static inline void qede_update_tx_producer(struct qede_tx_queue *txq) mmiowb(); } +static int qede_xdp_xmit(struct qede_dev *edev, struct qede_fastpath *fp, + struct sw_rx_data *metadata, u16 padding, u16 length) +{ + struct qede_tx_queue *txq = fp->xdp_tx; + u16 idx = txq->sw_tx_prod & NUM_TX_BDS_MAX; + struct eth_tx_1st_bd *first_bd; + + if (!qed_chain_get_elem_left(&txq->tx_pbl)) { + txq->stopped_cnt++; + return -ENOMEM; + } + + first_bd = (struct eth_tx_1st_bd *)qed_chain_produce(&txq->tx_pbl); + + memset(first_bd, 0, sizeof(*first_bd)); + first_bd->data.bd_flags.bitfields = + BIT(ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT); + first_bd->data.bitfields |= + (length & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) << + ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; + first_bd->data.nbds = 1; + + /* We can safely ignore the offset, as it's 0 for XDP */ + BD_SET_UNMAP_ADDR_LEN(first_bd, metadata->mapping + padding, length); + + /* Synchronize the buffer back to device, as program [probably] + * has changed it. + */ + dma_sync_single_for_device(&edev->pdev->dev, + metadata->mapping + padding, + length, PCI_DMA_TODEVICE); + + txq->sw_tx_ring.pages[idx] = metadata->data; + txq->sw_tx_prod++; + + /* Mark the fastpath for future XDP doorbell */ + fp->xdp_xmit = 1; + + return 0; +} + /* Main transmit function */ static netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev) @@ -573,7 +617,7 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb, /* Fill the entry in the SW ring and the BDs in the FW ring */ idx = txq->sw_tx_prod & NUM_TX_BDS_MAX; - txq->sw_tx_ring[idx].skb = skb; + txq->sw_tx_ring.skbs[idx].skb = skb; first_bd = (struct eth_tx_1st_bd *) qed_chain_produce(&txq->tx_pbl); memset(first_bd, 0, sizeof(*first_bd)); @@ -693,7 +737,7 @@ static netdev_tx_t qede_start_xmit(struct sk_buff *skb, /* this marks the BD as one that has no * individual mapping */ - txq->sw_tx_ring[idx].flags |= QEDE_TSO_SPLIT_BD; + txq->sw_tx_ring.skbs[idx].flags |= QEDE_TSO_SPLIT_BD; first_bd->nbytes = cpu_to_le16(hlen); @@ -802,6 +846,27 @@ int qede_txq_has_work(struct qede_tx_queue *txq) return hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl); } +static void qede_xdp_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq) +{ + struct eth_tx_1st_bd *bd; + u16 hw_bd_cons; + + hw_bd_cons = le16_to_cpu(*txq->hw_cons_ptr); + barrier(); + + while (hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl)) { + bd = (struct eth_tx_1st_bd *)qed_chain_consume(&txq->tx_pbl); + + dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(bd), + PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(txq->sw_tx_ring.pages[txq->sw_tx_cons & + NUM_TX_BDS_MAX]); + + txq->sw_tx_cons++; + txq->xmit_pkts++; + } +} + static int qede_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq) { struct netdev_queue *netdev_txq; @@ -942,7 +1007,7 @@ static int qede_alloc_rx_buffer(struct qede_rx_queue *rxq) * for multiple RX buffer segment size mapping. */ mapping = dma_map_page(rxq->dev, data, 0, - PAGE_SIZE, DMA_FROM_DEVICE); + PAGE_SIZE, rxq->data_direction); if (unlikely(dma_mapping_error(rxq->dev, mapping))) { __free_page(data); return -ENOMEM; @@ -981,7 +1046,7 @@ static inline int qede_realloc_rx_buffer(struct qede_rx_queue *rxq, } dma_unmap_page(rxq->dev, curr_cons->mapping, - PAGE_SIZE, DMA_FROM_DEVICE); + PAGE_SIZE, rxq->data_direction); } else { /* Increment refcount of the page as we don't want * network stack to take the ownership of the page @@ -1441,6 +1506,26 @@ static bool qede_rx_xdp(struct qede_dev *edev, rxq->xdp_no_pass++; switch (act) { + case XDP_TX: + /* We need the replacement buffer before transmit. */ + if (qede_alloc_rx_buffer(rxq)) { + qede_recycle_rx_bd_ring(rxq, 1); + return false; + } + + /* Now if there's a transmission problem, we'd still have to + * throw current buffer, as replacement was already allocated. + */ + if (qede_xdp_xmit(edev, fp, bd, cqe->placement_offset, len)) { + dma_unmap_page(rxq->dev, bd->mapping, + PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(bd->data); + } + + /* Regardless, we've consumed an Rx BD */ + qede_rx_bd_ring_consume(rxq); + return false; + default: bpf_warn_invalid_xdp_action(act); case XDP_ABORTED: @@ -1740,6 +1825,10 @@ static bool qede_poll_is_more_work(struct qede_fastpath *fp) if (qede_has_rx_work(fp->rxq)) return true; + if (fp->type & QEDE_FASTPATH_XDP) + if (qede_txq_has_work(fp->xdp_tx)) + return true; + if (likely(fp->type & QEDE_FASTPATH_TX)) if (qede_txq_has_work(fp->txq)) return true; @@ -1757,6 +1846,9 @@ static int qede_poll(struct napi_struct *napi, int budget) if (likely(fp->type & QEDE_FASTPATH_TX) && qede_txq_has_work(fp->txq)) qede_tx_int(edev, fp->txq); + if ((fp->type & QEDE_FASTPATH_XDP) && qede_txq_has_work(fp->xdp_tx)) + qede_xdp_tx_int(edev, fp->xdp_tx); + rx_work_done = (likely(fp->type & QEDE_FASTPATH_RX) && qede_has_rx_work(fp->rxq)) ? qede_rx_int(fp, budget) : 0; @@ -1771,6 +1863,14 @@ static int qede_poll(struct napi_struct *napi, int budget) } } + if (fp->xdp_xmit) { + u16 xdp_prod = qed_chain_get_prod_idx(&fp->xdp_tx->tx_pbl); + + fp->xdp_xmit = 0; + fp->xdp_tx->tx_db.data.bd_prod = cpu_to_le16(xdp_prod); + qede_update_tx_producer(fp->xdp_tx); + } + return rx_work_done; } @@ -2586,6 +2686,7 @@ static void qede_free_fp_array(struct qede_dev *edev) kfree(fp->sb_info); kfree(fp->rxq); + kfree(fp->xdp_tx); kfree(fp->txq); } kfree(edev->fp_array); @@ -2646,8 +2747,13 @@ static int qede_alloc_fp_array(struct qede_dev *edev) if (!fp->rxq) goto err; - if (edev->xdp_prog) + if (edev->xdp_prog) { + fp->xdp_tx = kzalloc(sizeof(*fp->xdp_tx), + GFP_KERNEL); + if (!fp->xdp_tx) + goto err; fp->type |= QEDE_FASTPATH_XDP; + } } } @@ -2694,9 +2800,9 @@ static void qede_update_pf_params(struct qed_dev *cdev) { struct qed_pf_params pf_params; - /* 64 rx + 64 tx */ + /* 64 rx + 64 tx + 64 XDP */ memset(&pf_params, 0, sizeof(struct qed_pf_params)); - pf_params.eth_pf_params.num_cons = 128; + pf_params.eth_pf_params.num_cons = 192; qed_ops->common->update_pf_params(cdev, &pf_params); } @@ -2953,7 +3059,7 @@ static void qede_free_rx_buffers(struct qede_dev *edev, data = rx_buf->data; dma_unmap_page(&edev->pdev->dev, - rx_buf->mapping, PAGE_SIZE, DMA_FROM_DEVICE); + rx_buf->mapping, PAGE_SIZE, rxq->data_direction); rx_buf->data = NULL; __free_page(data); @@ -3114,7 +3220,10 @@ err: static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) { /* Free the parallel SW ring */ - kfree(txq->sw_tx_ring); + if (txq->is_xdp) + kfree(txq->sw_tx_ring.pages); + else + kfree(txq->sw_tx_ring.skbs); /* Free the real RQ ring used by FW */ edev->ops->common->chain_free(edev->cdev, &txq->tx_pbl); @@ -3123,17 +3232,22 @@ static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) /* This function allocates all memory needed per Tx queue */ static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) { - int size, rc; union eth_tx_bd_types *p_virt; + int size, rc; txq->num_tx_buffers = edev->q_num_tx_buffers; /* Allocate the parallel driver ring for Tx buffers */ - size = sizeof(*txq->sw_tx_ring) * TX_RING_SIZE; - txq->sw_tx_ring = kzalloc(size, GFP_KERNEL); - if (!txq->sw_tx_ring) { - DP_NOTICE(edev, "Tx buffers ring allocation failed\n"); - goto err; + if (txq->is_xdp) { + size = sizeof(*txq->sw_tx_ring.pages) * TX_RING_SIZE; + txq->sw_tx_ring.pages = kzalloc(size, GFP_KERNEL); + if (!txq->sw_tx_ring.pages) + goto err; + } else { + size = sizeof(*txq->sw_tx_ring.skbs) * TX_RING_SIZE; + txq->sw_tx_ring.skbs = kzalloc(size, GFP_KERNEL); + if (!txq->sw_tx_ring.skbs) + goto err; } rc = edev->ops->common->chain_alloc(edev->cdev, @@ -3169,26 +3283,31 @@ static void qede_free_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp) */ static int qede_alloc_mem_fp(struct qede_dev *edev, struct qede_fastpath *fp) { - int rc; + int rc = 0; rc = qede_alloc_mem_sb(edev, fp->sb_info, fp->id); if (rc) - goto err; + goto out; if (fp->type & QEDE_FASTPATH_RX) { rc = qede_alloc_mem_rxq(edev, fp->rxq); if (rc) - goto err; + goto out; + } + + if (fp->type & QEDE_FASTPATH_XDP) { + rc = qede_alloc_mem_txq(edev, fp->xdp_tx); + if (rc) + goto out; } if (fp->type & QEDE_FASTPATH_TX) { rc = qede_alloc_mem_txq(edev, fp->txq); if (rc) - goto err; + goto out; } - return 0; -err: +out: return rc; } @@ -3236,9 +3355,20 @@ static void qede_init_fp(struct qede_dev *edev) fp->edev = edev; fp->id = queue_id; + if (fp->type & QEDE_FASTPATH_XDP) { + fp->xdp_tx->index = QEDE_TXQ_IDX_TO_XDP(edev, + rxq_index); + fp->xdp_tx->is_xdp = 1; + } if (fp->type & QEDE_FASTPATH_RX) { fp->rxq->rxq_id = rxq_index++; + + /* Determine how to map buffers for this queue */ + if (fp->type & QEDE_FASTPATH_XDP) + fp->rxq->data_direction = DMA_BIDIRECTIONAL; + else + fp->rxq->data_direction = DMA_FROM_DEVICE; fp->rxq->dev = &edev->pdev->dev; } @@ -3449,6 +3579,12 @@ static int qede_stop_queues(struct qede_dev *edev) if (rc) return rc; } + + if (fp->type & QEDE_FASTPATH_XDP) { + rc = qede_drain_txq(edev, fp->xdp_tx, true); + if (rc) + return rc; + } } /* Stop all Queues in reverse order */ @@ -3471,8 +3607,14 @@ static int qede_stop_queues(struct qede_dev *edev) } } - if (fp->type & QEDE_FASTPATH_XDP) + /* Stop the XDP forwarding queue */ + if (fp->type & QEDE_FASTPATH_XDP) { + rc = qede_stop_txq(edev, fp->xdp_tx, i); + if (rc) + return rc; + bpf_prog_put(fp->rxq->xdp_prog); + } } /* Stop the vport */ @@ -3496,7 +3638,14 @@ static int qede_start_txq(struct qede_dev *edev, memset(¶ms, 0, sizeof(params)); memset(&ret_params, 0, sizeof(ret_params)); - params.queue_id = txq->index; + /* Let the XDP queue share the queue-zone with one of the regular txq. + * We don't really care about its coalescing. + */ + if (txq->is_xdp) + params.queue_id = QEDE_TXQ_XDP_TO_IDX(edev, txq); + else + params.queue_id = txq->index; + params.sb = fp->sb_info->igu_sb_id; params.sb_idx = sb_idx; @@ -3601,6 +3750,10 @@ static int qede_start_queues(struct qede_dev *edev, bool clear_stats) } if (fp->type & QEDE_FASTPATH_XDP) { + rc = qede_start_txq(edev, fp, fp->xdp_tx, i, XDP_PI); + if (rc) + return rc; + fp->rxq->xdp_prog = bpf_prog_add(edev->xdp_prog, 1); if (IS_ERR(fp->rxq->xdp_prog)) { rc = PTR_ERR(fp->rxq->xdp_prog); -- 2.20.1