net: provide generic busy polling to all NAPI drivers

NAPI drivers no longer need to observe a particular protocol
to benefit from busy polling (CONFIG_NET_RX_BUSY_POLL=y)

napi_hash_add() and napi_hash_del() are automatically called
from core networking stack, respectively from
netif_napi_add() and netif_napi_del()

This patch depends on free_netdev() and netif_napi_del() being
called from process context, which seems to be the norm.

Drivers might still prefer to call napi_hash_del() on their
own, since they might combine all the rcu grace periods into
a single one, knowing their NAPI structures lifetime, while
core networking stack has no idea of a possible combining.

Once this patch proves to not bring serious regressions,
we will cleanup drivers to either remove napi_hash_del()
or provide appropriate rcu grace periods combining.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index ab9222924bd97a28656a2ec48944aa90ca652b16..d9add7c02e42ca89f4be202b390a24d704a746e4 100644 (file)
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -46,7 +46,6 @@ static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
        for_each_rx_queue_cnic(bp, i) {
                netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
                               bnx2x_poll, NAPI_POLL_WEIGHT);
-               napi_hash_add(&bnx2x_fp(bp, i, napi));
        }
 }
 
@@ -58,7 +57,6 @@ static void bnx2x_add_all_napi(struct bnx2x *bp)
        for_each_eth_queue(bp, i) {
                netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
                               bnx2x_poll, NAPI_POLL_WEIGHT);
-               napi_hash_add(&bnx2x_fp(bp, i, napi));
        }
 }
 

diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index db15c5ee09c53a528ea405961734dae927af0e06..f2d0dc9b1c4143704ea484be18c07a600752e4d1 100644 (file)
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -4227,12 +4227,10 @@ static void bnxt_init_napi(struct bnxt *bp)
                        bnapi = bp->bnapi[i];
                        netif_napi_add(bp->dev, &bnapi->napi,
                                       bnxt_poll, 64);
-                       napi_hash_add(&bnapi->napi);
                }
        } else {
                bnapi = bp->bnapi[0];
                netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
-               napi_hash_add(&bnapi->napi);
        }
 }
 

diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index f650f295f2646a80684ed5fcb8b794d4ed2b736f..48d8fbb1c2200f0f2c4a4f7a6935a7ccddf3c7a2 100644 (file)
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -2527,7 +2527,6 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
                goto err;
 
        netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
-       napi_hash_add(&iq->napi);
        iq->cur_desc = iq->desc;
        iq->cidx = 0;
        iq->gen = 1;

diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c
index b36643ef0593a1679dc0e0605711bb2846cfd58d..b2182d3ba3cc464d1504856e7d3d36712e9c65d5 100644 (file)
--- a/drivers/net/ethernet/cisco/enic/enic_main.c
+++ b/drivers/net/ethernet/cisco/enic/enic_main.c
@@ -2458,13 +2458,11 @@ static int enic_dev_init(struct enic *enic)
        switch (vnic_dev_get_intr_mode(enic->vdev)) {
        default:
                netif_napi_add(netdev, &enic->napi[0], enic_poll, 64);
-               napi_hash_add(&enic->napi[0]);
                break;
        case VNIC_DEV_INTR_MODE_MSIX:
                for (i = 0; i < enic->rq_count; i++) {
                        netif_napi_add(netdev, &enic->napi[i],
                                enic_poll_msix_rq, NAPI_POLL_WEIGHT);
-                       napi_hash_add(&enic->napi[i]);
                }
                for (i = 0; i < enic->wq_count; i++)
                        netif_napi_add(netdev, &enic->napi[enic_cq_wq(enic, i)],

diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index c29d62496ad990c5ed5df93b0957b19221134a9d..4cab8879f5ae4f0b3f64c9f7eb03c217f8e2ad19 100644 (file)
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -2630,7 +2630,6 @@ static int be_evt_queues_create(struct be_adapter *adapter)
                                eqo->affinity_mask);
                netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
                               BE_NAPI_WEIGHT);
-               napi_hash_add(&eqo->napi);
        }
        return 0;
 }

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index f3168bcc7d87905b2f3563ea64c0fbfeda01d1d4..e771e764daa3ae9b9390e2a49ae45f8cf2bafa7d 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -844,7 +844,6 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
        /* initialize NAPI */
        netif_napi_add(adapter->netdev, &q_vector->napi,
                       ixgbe_poll, 64);
-       napi_hash_add(&q_vector->napi);
 
 #ifdef CONFIG_NET_RX_BUSY_POLL
        /* initialize busy poll */

diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 592ff237d69207837358a38025e31f4e9989b540..2955186cd4f68332a308538616697b2f2910345f 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -2483,9 +2483,6 @@ static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
                q_vector->v_idx = q_idx;
                netif_napi_add(adapter->netdev, &q_vector->napi,
                               ixgbevf_poll, 64);
-#ifdef CONFIG_NET_RX_BUSY_POLL
-               napi_hash_add(&q_vector->napi);
-#endif
                adapter->q_vector[q_idx] = q_vector;
        }
 

diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
index 3a6176fea78d0657fb5314088685a25df7476890..af975a2b74c6d277c1a744cf358846d66de38abe 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
@@ -155,13 +155,11 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
        cq->mcq.comp  = cq->is_tx ? mlx4_en_tx_irq : mlx4_en_rx_irq;
        cq->mcq.event = mlx4_en_cq_event;
 
-       if (cq->is_tx) {
+       if (cq->is_tx)
                netif_tx_napi_add(cq->dev, &cq->napi, mlx4_en_poll_tx_cq,
                                  NAPI_POLL_WEIGHT);
-       } else {
+       else
                netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq, 64);
-               napi_hash_add(&cq->napi);
-       }
 
        napi_enable(&cq->napi);
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index ffb1f9c1b973042057b3b3ad879896dad0a0bbee..f6a8cc78760355b2ceb50ca05488926fca99759a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -982,7 +982,6 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
        mlx5e_build_channeltc_to_txq_map(priv, ix);
 
        netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
-       napi_hash_add(&c->napi);
 
        err = mlx5e_open_tx_cqs(c, cparam);
        if (err)

diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
index acf866147d65c0dc76bba0457916c6f20a9e06fe..270c9eeb7ab622955d237ae9cd96cadc5caa5f3a 100644 (file)
--- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
+++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
@@ -3814,7 +3814,6 @@ static int myri10ge_alloc_slices(struct myri10ge_priv *mgp)
                ss->dev = mgp->dev;
                netif_napi_add(ss->dev, &ss->napi, myri10ge_poll,
                               myri10ge_napi_weight);
-               napi_hash_add(&ss->napi);
        }
        return 0;
 abort:

diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index a3c42a376741f9c86179ca0fae5889203b9f16d5..4e82bcfbe3e09f837c827f72c94d9ae91f02e614 100644 (file)
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -2059,7 +2059,6 @@ static void efx_init_napi_channel(struct efx_channel *channel)
        channel->napi_dev = efx->net_dev;
        netif_napi_add(channel->napi_dev, &channel->napi_str,
                       efx_poll, napi_weight);
-       napi_hash_add(&channel->napi_str);
        efx_channel_busy_poll_init(channel);
 }
 

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index d1d14cecf45071bd16aad1bc1435f1963df6c081..b1ae4cbf24534f824d7bfd8977b2ef3dbfb6b9de 100644 (file)
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -1610,7 +1610,6 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
                vi->rq[i].pages = NULL;
                netif_napi_add(vi->dev, &vi->rq[i].napi, virtnet_poll,
                               napi_weight);
-               napi_hash_add(&vi->rq[i].napi);
 
                sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
                ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index e5c33b29471b70bd5bc9fc70af4b91a33758b234..7d2d1d7aaec7c5cdcaaea1d59f9e77adcca61597 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -466,6 +466,9 @@ static inline void napi_complete(struct napi_struct *n)
  *     @napi: napi context
  *
  * generate a new napi_id and store a @napi under it in napi_hash
+ * Used for busy polling (CONFIG_NET_RX_BUSY_POLL)
+ * Note: This is normally automatically done from netif_napi_add(),
+ * so might disappear in a future linux version.
  */
 void napi_hash_add(struct napi_struct *napi);
 
@@ -476,6 +479,10 @@ void napi_hash_add(struct napi_struct *napi);
  * Warning: caller must observe rcu grace period
  * before freeing memory containing @napi, if
  * this function returns true.
+ * Note: core networking stack automatically calls it
+ * from netif_napi_del()
+ * Drivers might want to call this helper to combine all
+ * the needed rcu grace periods into a single one.
  */
 bool napi_hash_del(struct napi_struct *napi);
 

diff --git a/net/core/dev.c b/net/core/dev.c
index 59dddac1c2e70c99b7879936e56041945ea9dbf7..41cef3e3f558b857a9093a83f6b0c73f32b8b45f 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4807,6 +4807,7 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
        napi->poll_owner = -1;
 #endif
        set_bit(NAPI_STATE_SCHED, &napi->state);
+       napi_hash_add(napi);
 }
 EXPORT_SYMBOL(netif_napi_add);
 
@@ -4826,8 +4827,12 @@ void napi_disable(struct napi_struct *n)
 }
 EXPORT_SYMBOL(napi_disable);
 
+/* Must be called in process context */
 void netif_napi_del(struct napi_struct *napi)
 {
+       might_sleep();
+       if (napi_hash_del(napi))
+               synchronize_net();
        list_del_init(&napi->dev_list);
        napi_free_frags(napi);
 
@@ -7227,11 +7232,13 @@ EXPORT_SYMBOL(alloc_netdev_mqs);
  *     This function does the last stage of destroying an allocated device
  *     interface. The reference to the device object is released.
  *     If this is the last reference then it will be freed.
+ *     Must be called in process context.
  */
 void free_netdev(struct net_device *dev)
 {
        struct napi_struct *p, *n;
 
+       might_sleep();
        netif_free_tx_queues(dev);
 #ifdef CONFIG_SYSFS
        kvfree(dev->_rx);