IB/IPoIB: Separate control from HW operation on ipoib_open/stop ndo
authorErez Shitrit <erezsh@mellanox.com>
Mon, 10 Apr 2017 08:22:27 +0000 (11:22 +0300)
committerDoug Ledford <dledford@redhat.com>
Thu, 20 Apr 2017 19:19:43 +0000 (15:19 -0400)
This patch is preparing the netdev part at the IPoIB driver to be able
to use the ipoib_options.

It deals with the two flows from the .ndo: ipoib_open and ipoib_stop.

The code is rearranged as follows:
 * All operations which deal with the hardware resources, (for example
   change QP state, post-receive etc.) are performed in one place.
 * All operations that are control oriented (like restart multicast task,
   start the reap_ah etc.) are performed in separate place.

The functions that deal with the hardware resources now located at
__ipoib_ib_dev_open for the ipoib_open flow and __ipoib_ib_dev_stop
for ipoib_stop.

Signed-off-by: Erez Shitrit <erezsh@mellanox.com>
Reviewed-by: Alex Vesker <valex@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>
drivers/infiniband/ulp/ipoib/ipoib.h
drivers/infiniband/ulp/ipoib/ipoib_ib.c
drivers/infiniband/ulp/ipoib/ipoib_main.c

index bc71a90c57e3ce561dbd0ae6fafe7bec89e6a1e7..596c73d01add6f5414fe68c5852f3768346ba5c0 100644 (file)
@@ -503,7 +503,7 @@ void ipoib_dev_uninit_default(struct net_device *dev);
 int ipoib_ib_dev_open(struct net_device *dev);
 void ipoib_ib_dev_up(struct net_device *dev);
 void ipoib_ib_dev_down(struct net_device *dev);
-void ipoib_ib_dev_stop(struct net_device *dev);
+int ipoib_ib_dev_stop_default(struct net_device *dev);
 void ipoib_pkey_dev_check_presence(struct net_device *dev);
 
 int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port);
index f9dd2bb676aa367750fffb3e234290d73892cb70..9d4f1b456ec4b08ae4fe6bc29c61e008b22e6399 100644 (file)
@@ -692,24 +692,115 @@ static void ipoib_stop_ah(struct net_device *dev)
        ipoib_flush_ah(dev);
 }
 
-void ipoib_ib_tx_timer_func(unsigned long ctx)
+static int recvs_pending(struct net_device *dev)
 {
-       drain_tx_cq((struct net_device *)ctx);
+       struct ipoib_dev_priv *priv = netdev_priv(dev);
+       int pending = 0;
+       int i;
+
+       for (i = 0; i < ipoib_recvq_size; ++i)
+               if (priv->rx_ring[i].skb)
+                       ++pending;
+
+       return pending;
 }
 
-int ipoib_ib_dev_open(struct net_device *dev)
+int ipoib_ib_dev_stop_default(struct net_device *dev)
 {
        struct ipoib_dev_priv *priv = netdev_priv(dev);
-       int ret;
+       struct ib_qp_attr qp_attr;
+       unsigned long begin;
+       struct ipoib_tx_buf *tx_req;
+       int i;
 
-       ipoib_pkey_dev_check_presence(dev);
+       if (test_and_clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
+               napi_disable(&priv->napi);
 
-       if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
-               ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,
-                          (!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));
-               return -1;
+       ipoib_cm_dev_stop(dev);
+
+       /*
+        * Move our QP to the error state and then reinitialize in
+        * when all work requests have completed or have been flushed.
+        */
+       qp_attr.qp_state = IB_QPS_ERR;
+       if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
+               ipoib_warn(priv, "Failed to modify QP to ERROR state\n");
+
+       /* Wait for all sends and receives to complete */
+       begin = jiffies;
+
+       while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
+               if (time_after(jiffies, begin + 5 * HZ)) {
+                       ipoib_warn(priv,
+                                  "timing out; %d sends %d receives not completed\n",
+                                  priv->tx_head - priv->tx_tail,
+                                  recvs_pending(dev));
+
+                       /*
+                        * assume the HW is wedged and just free up
+                        * all our pending work requests.
+                        */
+                       while ((int)priv->tx_tail - (int)priv->tx_head < 0) {
+                               tx_req = &priv->tx_ring[priv->tx_tail &
+                                                       (ipoib_sendq_size - 1)];
+                               ipoib_dma_unmap_tx(priv, tx_req);
+                               dev_kfree_skb_any(tx_req->skb);
+                               ++priv->tx_tail;
+                               --priv->tx_outstanding;
+                       }
+
+                       for (i = 0; i < ipoib_recvq_size; ++i) {
+                               struct ipoib_rx_buf *rx_req;
+
+                               rx_req = &priv->rx_ring[i];
+                               if (!rx_req->skb)
+                                       continue;
+                               ipoib_ud_dma_unmap_rx(priv,
+                                                     priv->rx_ring[i].mapping);
+                               dev_kfree_skb_any(rx_req->skb);
+                               rx_req->skb = NULL;
+                       }
+
+                       goto timeout;
+               }
+
+               ipoib_drain_cq(dev);
+
+               msleep(1);
        }
 
+       ipoib_dbg(priv, "All sends and receives done.\n");
+
+timeout:
+       del_timer_sync(&priv->poll_timer);
+       qp_attr.qp_state = IB_QPS_RESET;
+       if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
+               ipoib_warn(priv, "Failed to modify QP to RESET state\n");
+
+       ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
+
+       return 0;
+}
+
+int ipoib_ib_dev_stop(struct net_device *dev)
+{
+       ipoib_ib_dev_stop_default(dev);
+
+       ipoib_flush_ah(dev);
+
+       return 0;
+}
+
+void ipoib_ib_tx_timer_func(unsigned long ctx)
+{
+       drain_tx_cq((struct net_device *)ctx);
+}
+
+int ipoib_ib_dev_open_default(struct net_device *dev)
+{
+       struct ipoib_dev_priv *priv = netdev_priv(dev);
+       int ret;
+
        ret = ipoib_init_qp(dev);
        if (ret) {
                ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
@@ -728,10 +819,6 @@ int ipoib_ib_dev_open(struct net_device *dev)
                goto dev_stop;
        }
 
-       clear_bit(IPOIB_STOP_REAPER, &priv->flags);
-       queue_delayed_work(priv->wq, &priv->ah_reap_task,
-                          round_jiffies_relative(HZ));
-
        if (!test_and_set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
                napi_enable(&priv->napi);
 
@@ -743,6 +830,35 @@ dev_stop:
        return -1;
 }
 
+int ipoib_ib_dev_open(struct net_device *dev)
+{
+       struct ipoib_dev_priv *priv = netdev_priv(dev);
+
+       ipoib_pkey_dev_check_presence(dev);
+
+       if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
+               ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,
+                          (!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));
+               return -1;
+       }
+
+       clear_bit(IPOIB_STOP_REAPER, &priv->flags);
+       queue_delayed_work(priv->wq, &priv->ah_reap_task,
+                          round_jiffies_relative(HZ));
+
+       if (ipoib_ib_dev_open_default(dev)) {
+               pr_warn("%s: Failed to open dev\n", dev->name);
+               goto stop_ah_reap;
+       }
+
+       return 0;
+
+stop_ah_reap:
+       set_bit(IPOIB_STOP_REAPER, &priv->flags);
+       cancel_delayed_work(&priv->ah_reap_task);
+       return -1;
+}
+
 void ipoib_pkey_dev_check_presence(struct net_device *dev)
 {
        struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -786,19 +902,6 @@ void ipoib_ib_dev_down(struct net_device *dev)
        ipoib_flush_paths(dev);
 }
 
-static int recvs_pending(struct net_device *dev)
-{
-       struct ipoib_dev_priv *priv = netdev_priv(dev);
-       int pending = 0;
-       int i;
-
-       for (i = 0; i < ipoib_recvq_size; ++i)
-               if (priv->rx_ring[i].skb)
-                       ++pending;
-
-       return pending;
-}
-
 void ipoib_drain_cq(struct net_device *dev)
 {
        struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -838,81 +941,6 @@ void ipoib_drain_cq(struct net_device *dev)
        local_bh_enable();
 }
 
-void ipoib_ib_dev_stop(struct net_device *dev)
-{
-       struct ipoib_dev_priv *priv = netdev_priv(dev);
-       struct ib_qp_attr qp_attr;
-       unsigned long begin;
-       struct ipoib_tx_buf *tx_req;
-       int i;
-
-       if (test_and_clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
-               napi_disable(&priv->napi);
-
-       ipoib_cm_dev_stop(dev);
-
-       /*
-        * Move our QP to the error state and then reinitialize in
-        * when all work requests have completed or have been flushed.
-        */
-       qp_attr.qp_state = IB_QPS_ERR;
-       if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
-               ipoib_warn(priv, "Failed to modify QP to ERROR state\n");
-
-       /* Wait for all sends and receives to complete */
-       begin = jiffies;
-
-       while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
-               if (time_after(jiffies, begin + 5 * HZ)) {
-                       ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",
-                                  priv->tx_head - priv->tx_tail, recvs_pending(dev));
-
-                       /*
-                        * assume the HW is wedged and just free up
-                        * all our pending work requests.
-                        */
-                       while ((int) priv->tx_tail - (int) priv->tx_head < 0) {
-                               tx_req = &priv->tx_ring[priv->tx_tail &
-                                                       (ipoib_sendq_size - 1)];
-                               ipoib_dma_unmap_tx(priv, tx_req);
-                               dev_kfree_skb_any(tx_req->skb);
-                               ++priv->tx_tail;
-                               --priv->tx_outstanding;
-                       }
-
-                       for (i = 0; i < ipoib_recvq_size; ++i) {
-                               struct ipoib_rx_buf *rx_req;
-
-                               rx_req = &priv->rx_ring[i];
-                               if (!rx_req->skb)
-                                       continue;
-                               ipoib_ud_dma_unmap_rx(priv,
-                                                     priv->rx_ring[i].mapping);
-                               dev_kfree_skb_any(rx_req->skb);
-                               rx_req->skb = NULL;
-                       }
-
-                       goto timeout;
-               }
-
-               ipoib_drain_cq(dev);
-
-               msleep(1);
-       }
-
-       ipoib_dbg(priv, "All sends and receives done.\n");
-
-timeout:
-       del_timer_sync(&priv->poll_timer);
-       qp_attr.qp_state = IB_QPS_RESET;
-       if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
-               ipoib_warn(priv, "Failed to modify QP to RESET state\n");
-
-       ipoib_flush_ah(dev);
-
-       ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
-}
-
 /*
  * Takes whatever value which is in pkey index 0 and updates priv->pkey
  * returns 0 if the pkey value was changed.
index a0dfe471ab50df7b01dd62211d3ac98291c1bb86..8422850c4907fa353eafc1a85f869638b427f8b0 100644 (file)
@@ -193,7 +193,7 @@ static int ipoib_stop(struct net_device *dev)
        netif_stop_queue(dev);
 
        ipoib_ib_dev_down(dev);
-       ipoib_ib_dev_stop(dev);
+       ipoib_ib_dev_stop_default(dev);
 
        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                struct ipoib_dev_priv *cpriv;