From 74608d17fe29b2cddceea609033019b32e8a0650 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Bj=C3=B6rn=20T=C3=B6pel?= Date: Wed, 24 May 2017 07:55:35 +0200 Subject: [PATCH] i40e: add support for XDP_TX action MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This patch adds proper XDP_TX action support. For each Tx ring, an additional XDP Tx ring is allocated and setup. This version does the DMA mapping in the fast-path, which will penalize performance for IOMMU enabled systems. Further, debugfs support is not wired up for the XDP Tx rings. Signed-off-by: Björn Töpel Tested-by: Andrew Bowers Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/i40e/i40e.h | 1 + .../net/ethernet/intel/i40e/i40e_ethtool.c | 42 ++- drivers/net/ethernet/intel/i40e/i40e_main.c | 299 +++++++++++++----- drivers/net/ethernet/intel/i40e/i40e_txrx.c | 118 ++++++- drivers/net/ethernet/intel/i40e/i40e_txrx.h | 11 + 5 files changed, 384 insertions(+), 87 deletions(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index d3195b29d53c..4250ab55a9f1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -629,6 +629,7 @@ struct i40e_vsi { /* These are containers of ring pointers, allocated at run-time */ struct i40e_ring **rx_rings; struct i40e_ring **tx_rings; + struct i40e_ring **xdp_rings; /* XDP Tx rings */ u32 active_filters; u32 promisc_threshold; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 3d58762efbc0..9d3233c2c9cd 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -1299,6 +1299,17 @@ static void i40e_get_ringparam(struct net_device *netdev, ring->rx_jumbo_pending = 0; } +static bool i40e_active_tx_ring_index(struct i40e_vsi *vsi, u16 index) +{ + if (i40e_enabled_xdp_vsi(vsi)) { + return index < vsi->num_queue_pairs || + (index >= vsi->alloc_queue_pairs && + index < vsi->alloc_queue_pairs + vsi->num_queue_pairs); + } + + return index < vsi->num_queue_pairs; +} + static int i40e_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) { @@ -1308,6 +1319,7 @@ static int i40e_set_ringparam(struct net_device *netdev, struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; u32 new_rx_count, new_tx_count; + u16 tx_alloc_queue_pairs; int timeout = 50; int i, err = 0; @@ -1345,6 +1357,8 @@ static int i40e_set_ringparam(struct net_device *netdev, for (i = 0; i < vsi->num_queue_pairs; i++) { vsi->tx_rings[i]->count = new_tx_count; vsi->rx_rings[i]->count = new_rx_count; + if (i40e_enabled_xdp_vsi(vsi)) + vsi->xdp_rings[i]->count = new_tx_count; } goto done; } @@ -1354,20 +1368,24 @@ static int i40e_set_ringparam(struct net_device *netdev, * to the Tx and Rx ring structs. */ - /* alloc updated Tx resources */ + /* alloc updated Tx and XDP Tx resources */ + tx_alloc_queue_pairs = vsi->alloc_queue_pairs * + (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); if (new_tx_count != vsi->tx_rings[0]->count) { netdev_info(netdev, "Changing Tx descriptor count from %d to %d.\n", vsi->tx_rings[0]->count, new_tx_count); - tx_rings = kcalloc(vsi->alloc_queue_pairs, + tx_rings = kcalloc(tx_alloc_queue_pairs, sizeof(struct i40e_ring), GFP_KERNEL); if (!tx_rings) { err = -ENOMEM; goto done; } - for (i = 0; i < vsi->num_queue_pairs; i++) { - /* clone ring and setup updated count */ + for (i = 0; i < tx_alloc_queue_pairs; i++) { + if (!i40e_active_tx_ring_index(vsi, i)) + continue; + tx_rings[i] = *vsi->tx_rings[i]; tx_rings[i].count = new_tx_count; /* the desc and bi pointers will be reallocated in the @@ -1379,6 +1397,8 @@ static int i40e_set_ringparam(struct net_device *netdev, if (err) { while (i) { i--; + if (!i40e_active_tx_ring_index(vsi, i)) + continue; i40e_free_tx_resources(&tx_rings[i]); } kfree(tx_rings); @@ -1446,9 +1466,11 @@ rx_unwind: i40e_down(vsi); if (tx_rings) { - for (i = 0; i < vsi->num_queue_pairs; i++) { - i40e_free_tx_resources(vsi->tx_rings[i]); - *vsi->tx_rings[i] = tx_rings[i]; + for (i = 0; i < tx_alloc_queue_pairs; i++) { + if (i40e_active_tx_ring_index(vsi, i)) { + i40e_free_tx_resources(vsi->tx_rings[i]); + *vsi->tx_rings[i] = tx_rings[i]; + } } kfree(tx_rings); tx_rings = NULL; @@ -1479,8 +1501,10 @@ rx_unwind: free_tx: /* error cleanup if the Rx allocations failed after getting Tx */ if (tx_rings) { - for (i = 0; i < vsi->num_queue_pairs; i++) - i40e_free_tx_resources(&tx_rings[i]); + for (i = 0; i < tx_alloc_queue_pairs; i++) { + if (i40e_active_tx_ring_index(vsi, i)) + i40e_free_tx_resources(vsi->tx_rings[i]); + } kfree(tx_rings); tx_rings = NULL; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 89bbe32a5934..7e415bb5a7dc 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -407,6 +407,27 @@ struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) return &vsi->net_stats; } +/** + * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring + * @ring: Tx ring to get statistics from + * @stats: statistics entry to be updated + **/ +static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, + struct rtnl_link_stats64 *stats) +{ + u64 bytes, packets; + unsigned int start; + + do { + start = u64_stats_fetch_begin_irq(&ring->syncp); + packets = ring->stats.packets; + bytes = ring->stats.bytes; + } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + + stats->tx_packets += packets; + stats->tx_bytes += bytes; +} + /** * i40e_get_netdev_stats_struct - Get statistics for netdev interface * @netdev: network interface device structure @@ -437,15 +458,8 @@ static void i40e_get_netdev_stats_struct(struct net_device *netdev, tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); if (!tx_ring) continue; + i40e_get_netdev_stats_struct_tx(tx_ring, stats); - do { - start = u64_stats_fetch_begin_irq(&tx_ring->syncp); - packets = tx_ring->stats.packets; - bytes = tx_ring->stats.bytes; - } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); - - stats->tx_packets += packets; - stats->tx_bytes += bytes; rx_ring = &tx_ring[1]; do { @@ -456,6 +470,9 @@ static void i40e_get_netdev_stats_struct(struct net_device *netdev, stats->rx_packets += packets; stats->rx_bytes += bytes; + + if (i40e_enabled_xdp_vsi(vsi)) + i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats); } rcu_read_unlock(); @@ -2814,6 +2831,12 @@ static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) for (i = 0; i < vsi->num_queue_pairs && !err; i++) err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); + if (!i40e_enabled_xdp_vsi(vsi)) + return err; + + for (i = 0; i < vsi->num_queue_pairs && !err; i++) + err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); + return err; } @@ -2827,12 +2850,17 @@ static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) { int i; - if (!vsi->tx_rings) - return; + if (vsi->tx_rings) { + for (i = 0; i < vsi->num_queue_pairs; i++) + if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) + i40e_free_tx_resources(vsi->tx_rings[i]); + } - for (i = 0; i < vsi->num_queue_pairs; i++) - if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) - i40e_free_tx_resources(vsi->tx_rings[i]); + if (vsi->xdp_rings) { + for (i = 0; i < vsi->num_queue_pairs; i++) + if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) + i40e_free_tx_resources(vsi->xdp_rings[i]); + } } /** @@ -3093,6 +3121,12 @@ static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) err = i40e_configure_tx_ring(vsi->tx_rings[i]); + if (!i40e_enabled_xdp_vsi(vsi)) + return err; + + for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) + err = i40e_configure_tx_ring(vsi->xdp_rings[i]); + return err; } @@ -3237,6 +3271,7 @@ static int i40e_vsi_configure(struct i40e_vsi *vsi) **/ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) { + bool has_xdp = i40e_enabled_xdp_vsi(vsi); struct i40e_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; u16 vector; @@ -3267,28 +3302,40 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) /* Linked list for the queuepairs assigned to this vector */ wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); for (q = 0; q < q_vector->num_ringpairs; q++) { + u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; u32 val; val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | - (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | - (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| - (I40E_QUEUE_TYPE_TX - << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); + (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | + (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_TX << + I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); wr32(hw, I40E_QINT_RQCTL(qp), val); + if (has_xdp) { + val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | + (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | + (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_TX << + I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + + wr32(hw, I40E_QINT_TQCTL(nextqp), val); + } + val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | - (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | - (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | - ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)| - (I40E_QUEUE_TYPE_RX - << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | + (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | + ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | + (I40E_QUEUE_TYPE_RX << + I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); /* Terminate the linked list */ if (q == (q_vector->num_ringpairs - 1)) - val |= (I40E_QUEUE_END_OF_LIST - << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); + val |= (I40E_QUEUE_END_OF_LIST << + I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); wr32(hw, I40E_QINT_TQCTL(qp), val); qp++; @@ -3342,6 +3389,7 @@ static void i40e_enable_misc_int_causes(struct i40e_pf *pf) **/ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) { + u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; struct i40e_q_vector *q_vector = vsi->q_vectors[0]; struct i40e_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; @@ -3362,12 +3410,22 @@ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) wr32(hw, I40E_PFINT_LNKLST0, 0); /* Associate the queue pair to the vector and enable the queue int */ - val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | + val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | + (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | + (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); wr32(hw, I40E_QINT_RQCTL(0), val); + if (i40e_enabled_xdp_vsi(vsi)) { + val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | + (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| + (I40E_QUEUE_TYPE_TX + << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); + + wr32(hw, I40E_QINT_TQCTL(nextqp), val); + } + val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); @@ -3534,6 +3592,9 @@ static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) for (i = 0; i < vsi->num_queue_pairs; i++) { wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0); wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0); + if (!i40e_enabled_xdp_vsi(vsi)) + continue; + wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); } if (pf->flags & I40E_FLAG_MSIX_ENABLED) { @@ -3836,6 +3897,16 @@ static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) q_vector->tx.ring = tx_ring; q_vector->tx.count++; + /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ + if (i40e_enabled_xdp_vsi(vsi)) { + struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; + + xdp_ring->q_vector = q_vector; + xdp_ring->next = q_vector->tx.ring; + q_vector->tx.ring = xdp_ring; + q_vector->tx.count++; + } + rx_ring->q_vector = q_vector; rx_ring->next = q_vector->rx.ring; q_vector->rx.ring = rx_ring; @@ -4014,6 +4085,33 @@ static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); } +/** + * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion + * @seid: VSI SEID + * @pf: the PF structure + * @pf_q: the PF queue to configure + * @is_xdp: true if the queue is used for XDP + * @enable: start or stop the queue + **/ +static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, + bool is_xdp, bool enable) +{ + int ret; + + i40e_control_tx_q(pf, pf_q, enable); + + /* wait for the change to finish */ + ret = i40e_pf_txq_wait(pf, pf_q, enable); + if (ret) { + dev_info(&pf->pdev->dev, + "VSI seid %d %sTx ring %d %sable timeout\n", + seid, (is_xdp ? "XDP " : ""), pf_q, + (enable ? "en" : "dis")); + } + + return ret; +} + /** * i40e_vsi_control_tx - Start or stop a VSI's rings * @vsi: the VSI being configured @@ -4026,16 +4124,20 @@ static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) pf_q = vsi->base_queue; for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { - i40e_control_tx_q(pf, pf_q, enable); + ret = i40e_control_wait_tx_q(vsi->seid, pf, + pf_q, + false /*is xdp*/, enable); + if (ret) + break; - /* wait for the change to finish */ - ret = i40e_pf_txq_wait(pf, pf_q, enable); - if (ret) { - dev_info(&pf->pdev->dev, - "VSI seid %d Tx ring %d %sable timeout\n", - vsi->seid, pf_q, (enable ? "en" : "dis")); + if (!i40e_enabled_xdp_vsi(vsi)) + continue; + + ret = i40e_control_wait_tx_q(vsi->seid, pf, + pf_q + vsi->alloc_queue_pairs, + true /*is xdp*/, enable); + if (ret) break; - } } return ret; @@ -4547,7 +4649,21 @@ int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) vsi->seid, pf_q); return ret; } - /* Check and wait for the Tx queue */ + + if (!i40e_enabled_xdp_vsi(vsi)) + goto wait_rx; + + /* Check and wait for the XDP Tx queue */ + ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, + false); + if (ret) { + dev_info(&pf->pdev->dev, + "VSI seid %d XDP Tx ring %d disable timeout\n", + vsi->seid, pf_q); + return ret; + } +wait_rx: + /* Check and wait for the Rx queue */ ret = i40e_pf_rxq_wait(pf, pf_q, false); if (ret) { dev_info(&pf->pdev->dev, @@ -5466,6 +5582,8 @@ void i40e_down(struct i40e_vsi *vsi) for (i = 0; i < vsi->num_queue_pairs; i++) { i40e_clean_tx_ring(vsi->tx_rings[i]); + if (i40e_enabled_xdp_vsi(vsi)) + i40e_clean_tx_ring(vsi->xdp_rings[i]); i40e_clean_rx_ring(vsi->rx_rings[i]); } @@ -7535,15 +7653,22 @@ static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) **/ static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) { + struct i40e_ring **next_rings; int size; int ret = 0; - /* allocate memory for both Tx and Rx ring pointers */ - size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2; + /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ + size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * + (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); vsi->tx_rings = kzalloc(size, GFP_KERNEL); if (!vsi->tx_rings) return -ENOMEM; - vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs]; + next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; + if (i40e_enabled_xdp_vsi(vsi)) { + vsi->xdp_rings = next_rings; + next_rings += vsi->alloc_queue_pairs; + } + vsi->rx_rings = next_rings; if (alloc_qvectors) { /* allocate memory for q_vector pointers */ @@ -7663,6 +7788,7 @@ static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) kfree(vsi->tx_rings); vsi->tx_rings = NULL; vsi->rx_rings = NULL; + vsi->xdp_rings = NULL; } /** @@ -7746,6 +7872,8 @@ static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) kfree_rcu(vsi->tx_rings[i], rcu); vsi->tx_rings[i] = NULL; vsi->rx_rings[i] = NULL; + if (vsi->xdp_rings) + vsi->xdp_rings[i] = NULL; } } } @@ -7756,43 +7884,61 @@ static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) **/ static int i40e_alloc_rings(struct i40e_vsi *vsi) { - struct i40e_ring *tx_ring, *rx_ring; + int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; struct i40e_pf *pf = vsi->back; - int i; + struct i40e_ring *ring; /* Set basic values in the rings to be used later during open() */ for (i = 0; i < vsi->alloc_queue_pairs; i++) { /* allocate space for both Tx and Rx in one shot */ - tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL); - if (!tx_ring) + ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); + if (!ring) goto err_out; - tx_ring->queue_index = i; - tx_ring->reg_idx = vsi->base_queue + i; - tx_ring->ring_active = false; - tx_ring->vsi = vsi; - tx_ring->netdev = vsi->netdev; - tx_ring->dev = &pf->pdev->dev; - tx_ring->count = vsi->num_desc; - tx_ring->size = 0; - tx_ring->dcb_tc = 0; + ring->queue_index = i; + ring->reg_idx = vsi->base_queue + i; + ring->ring_active = false; + ring->vsi = vsi; + ring->netdev = vsi->netdev; + ring->dev = &pf->pdev->dev; + ring->count = vsi->num_desc; + ring->size = 0; + ring->dcb_tc = 0; if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) - tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; - tx_ring->tx_itr_setting = pf->tx_itr_default; - vsi->tx_rings[i] = tx_ring; - - rx_ring = &tx_ring[1]; - rx_ring->queue_index = i; - rx_ring->reg_idx = vsi->base_queue + i; - rx_ring->ring_active = false; - rx_ring->vsi = vsi; - rx_ring->netdev = vsi->netdev; - rx_ring->dev = &pf->pdev->dev; - rx_ring->count = vsi->num_desc; - rx_ring->size = 0; - rx_ring->dcb_tc = 0; - rx_ring->rx_itr_setting = pf->rx_itr_default; - vsi->rx_rings[i] = rx_ring; + ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; + ring->tx_itr_setting = pf->tx_itr_default; + vsi->tx_rings[i] = ring++; + + if (!i40e_enabled_xdp_vsi(vsi)) + goto setup_rx; + + ring->queue_index = vsi->alloc_queue_pairs + i; + ring->reg_idx = vsi->base_queue + ring->queue_index; + ring->ring_active = false; + ring->vsi = vsi; + ring->netdev = NULL; + ring->dev = &pf->pdev->dev; + ring->count = vsi->num_desc; + ring->size = 0; + ring->dcb_tc = 0; + if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) + ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; + set_ring_xdp(ring); + ring->tx_itr_setting = pf->tx_itr_default; + vsi->xdp_rings[i] = ring++; + +setup_rx: + ring->queue_index = i; + ring->reg_idx = vsi->base_queue + i; + ring->ring_active = false; + ring->vsi = vsi; + ring->netdev = vsi->netdev; + ring->dev = &pf->pdev->dev; + ring->count = vsi->num_desc; + ring->size = 0; + ring->dcb_tc = 0; + ring->rx_itr_setting = pf->rx_itr_default; + vsi->rx_rings[i] = ring; } return 0; @@ -9998,6 +10144,7 @@ vector_setup_out: **/ static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) { + u16 alloc_queue_pairs; struct i40e_pf *pf; u8 enabled_tc; int ret; @@ -10016,11 +10163,14 @@ static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) if (ret) goto err_vsi; - ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx); + alloc_queue_pairs = vsi->alloc_queue_pairs * + (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); + + ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); if (ret < 0) { dev_info(&pf->pdev->dev, "failed to get tracking for %d queues for VSI %d err %d\n", - vsi->alloc_queue_pairs, vsi->seid, ret); + alloc_queue_pairs, vsi->seid, ret); goto err_vsi; } vsi->base_queue = ret; @@ -10076,6 +10226,7 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, { struct i40e_vsi *vsi = NULL; struct i40e_veb *veb = NULL; + u16 alloc_queue_pairs; int ret, i; int v_idx; @@ -10163,12 +10314,14 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, else if (type == I40E_VSI_SRIOV) vsi->vf_id = param1; /* assign it some queues */ - ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, - vsi->idx); + alloc_queue_pairs = vsi->alloc_queue_pairs * + (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); + + ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); if (ret < 0) { dev_info(&pf->pdev->dev, "failed to get tracking for %d queues for VSI %d err=%d\n", - vsi->alloc_queue_pairs, vsi->seid, ret); + alloc_queue_pairs, vsi->seid, ret); goto err_vsi; } vsi->base_queue = ret; diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index f744f843bc72..b936febc315a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -630,6 +630,8 @@ static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring, if (tx_buffer->skb) { if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB) kfree(tx_buffer->raw_buf); + else if (ring_is_xdp(ring)) + page_frag_free(tx_buffer->raw_buf); else dev_kfree_skb_any(tx_buffer->skb); if (dma_unmap_len(tx_buffer, len)) @@ -771,8 +773,11 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, total_bytes += tx_buf->bytecount; total_packets += tx_buf->gso_segs; - /* free the skb */ - napi_consume_skb(tx_buf->skb, napi_budget); + /* free the skb/XDP data */ + if (ring_is_xdp(tx_ring)) + page_frag_free(tx_buf->raw_buf); + else + napi_consume_skb(tx_buf->skb, napi_budget); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -848,6 +853,9 @@ static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, tx_ring->arm_wb = true; } + if (ring_is_xdp(tx_ring)) + return !!budget; + /* notify netdev of completed buffers */ netdev_tx_completed_queue(txring_txq(tx_ring), total_packets, total_bytes); @@ -1969,6 +1977,10 @@ static bool i40e_is_non_eop(struct i40e_ring *rx_ring, #define I40E_XDP_PASS 0 #define I40E_XDP_CONSUMED 1 +#define I40E_XDP_TX 2 + +static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, + struct i40e_ring *xdp_ring); /** * i40e_run_xdp - run an XDP program @@ -1979,6 +1991,7 @@ static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp) { int result = I40E_XDP_PASS; + struct i40e_ring *xdp_ring; struct bpf_prog *xdp_prog; u32 act; @@ -1992,9 +2005,12 @@ static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring, switch (act) { case XDP_PASS: break; + case XDP_TX: + xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; + result = i40e_xmit_xdp_ring(xdp, xdp_ring); + break; default: bpf_warn_invalid_xdp_action(act); - case XDP_TX: case XDP_ABORTED: trace_xdp_exception(rx_ring->netdev, xdp_prog, act); /* fallthrough -- handle aborts by dropping packet */ @@ -2007,6 +2023,27 @@ xdp_out: return ERR_PTR(-result); } +/** + * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region + * @rx_ring: Rx ring + * @rx_buffer: Rx buffer to adjust + * @size: Size of adjustment + **/ +static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring, + struct i40e_rx_buffer *rx_buffer, + unsigned int size) +{ +#if (PAGE_SIZE < 8192) + unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; + + rx_buffer->page_offset ^= truesize; +#else + unsigned int truesize = SKB_DATA_ALIGN(i40e_rx_offset(rx_ring) + size); + + rx_buffer->page_offset += truesize; +#endif +} + /** * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @rx_ring: rx descriptor ring to transact packets on @@ -2024,7 +2061,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct sk_buff *skb = rx_ring->skb; u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); - bool failure = false; + bool failure = false, xdp_xmit = false; while (likely(total_rx_packets < budget)) { struct i40e_rx_buffer *rx_buffer; @@ -2081,9 +2118,14 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) } if (IS_ERR(skb)) { + if (PTR_ERR(skb) == -I40E_XDP_TX) { + xdp_xmit = true; + i40e_rx_buffer_flip(rx_ring, rx_buffer, size); + } else { + rx_buffer->pagecnt_bias++; + } total_rx_bytes += size; total_rx_packets++; - rx_buffer->pagecnt_bias++; } else if (skb) { i40e_add_rx_frag(rx_ring, rx_buffer, skb, size); } else if (ring_uses_build_skb(rx_ring)) { @@ -2131,6 +2173,19 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) total_rx_packets++; } + if (xdp_xmit) { + struct i40e_ring *xdp_ring; + + xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; + + /* Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. + */ + wmb(); + + writel(xdp_ring->next_to_use, xdp_ring->tail); + } + rx_ring->skb = skb; u64_stats_update_begin(&rx_ring->syncp); @@ -3187,6 +3242,59 @@ dma_error: return -1; } +/** + * i40e_xmit_xdp_ring - transmits an XDP buffer to an XDP Tx ring + * @xdp: data to transmit + * @xdp_ring: XDP Tx ring + **/ +static int i40e_xmit_xdp_ring(struct xdp_buff *xdp, + struct i40e_ring *xdp_ring) +{ + u32 size = xdp->data_end - xdp->data; + u16 i = xdp_ring->next_to_use; + struct i40e_tx_buffer *tx_bi; + struct i40e_tx_desc *tx_desc; + dma_addr_t dma; + + if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { + xdp_ring->tx_stats.tx_busy++; + return I40E_XDP_CONSUMED; + } + + dma = dma_map_single(xdp_ring->dev, xdp->data, size, DMA_TO_DEVICE); + if (dma_mapping_error(xdp_ring->dev, dma)) + return I40E_XDP_CONSUMED; + + tx_bi = &xdp_ring->tx_bi[i]; + tx_bi->bytecount = size; + tx_bi->gso_segs = 1; + tx_bi->raw_buf = xdp->data; + + /* record length, and DMA address */ + dma_unmap_len_set(tx_bi, len, size); + dma_unmap_addr_set(tx_bi, dma, dma); + + tx_desc = I40E_TX_DESC(xdp_ring, i); + tx_desc->buffer_addr = cpu_to_le64(dma); + tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC + | I40E_TXD_CMD, + 0, size, 0); + + /* Make certain all of the status bits have been updated + * before next_to_watch is written. + */ + smp_wmb(); + + i++; + if (i == xdp_ring->count) + i = 0; + + tx_bi->next_to_watch = tx_desc; + xdp_ring->next_to_use = i; + + return I40E_XDP_TX; +} + /** * i40e_xmit_frame_ring - Sends buffer on Tx ring * @skb: send buffer diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h index 31f0b162996f..b288d58313a6 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h @@ -396,6 +396,7 @@ struct i40e_ring { u16 flags; #define I40E_TXR_FLAGS_WB_ON_ITR BIT(0) #define I40E_RXR_FLAGS_BUILD_SKB_ENABLED BIT(1) +#define I40E_TXR_FLAGS_XDP BIT(2) /* stats structs */ struct i40e_queue_stats stats; @@ -438,6 +439,16 @@ static inline void clear_ring_build_skb_enabled(struct i40e_ring *ring) ring->flags &= ~I40E_RXR_FLAGS_BUILD_SKB_ENABLED; } +static inline bool ring_is_xdp(struct i40e_ring *ring) +{ + return !!(ring->flags & I40E_TXR_FLAGS_XDP); +} + +static inline void set_ring_xdp(struct i40e_ring *ring) +{ + ring->flags |= I40E_TXR_FLAGS_XDP; +} + enum i40e_latency_range { I40E_LOWEST_LATENCY = 0, I40E_LOW_LATENCY = 1, -- 2.20.1