amd-xgbe: Add support for per DMA channel interrupts
authorLendacky, Thomas <Thomas.Lendacky@amd.com>
Tue, 4 Nov 2014 22:06:56 +0000 (16:06 -0600)
committerDavid S. Miller <davem@davemloft.net>
Thu, 6 Nov 2014 02:50:12 +0000 (21:50 -0500)
This patch provides support for interrupts that are generated by the
Tx/Rx DMA channel pairs of the device.  This allows for Tx and Rx
processing to run across multiple processsors.

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Documentation/devicetree/bindings/net/amd-xgbe.txt
drivers/net/ethernet/amd/xgbe/xgbe-dev.c
drivers/net/ethernet/amd/xgbe/xgbe-drv.c
drivers/net/ethernet/amd/xgbe/xgbe-main.c
drivers/net/ethernet/amd/xgbe/xgbe.h

index 41354f730beba333d1921d428972459aacbd65cd..26efd526d16c29cc45de438aef24adb8f34898bd 100644 (file)
@@ -7,7 +7,10 @@ Required properties:
    - PCS registers
 - interrupt-parent: Should be the phandle for the interrupt controller
   that services interrupts for this device
-- interrupts: Should contain the amd-xgbe interrupt
+- interrupts: Should contain the amd-xgbe interrupt(s). The first interrupt
+  listed is required and is the general device interrupt. If the optional
+  amd,per-channel-interrupt property is specified, then one additional
+  interrupt for each DMA channel supported by the device should be specified
 - clocks:
    - DMA clock for the amd-xgbe device (used for calculating the
      correct Rx interrupt watchdog timer value on a DMA channel
@@ -23,6 +26,9 @@ Optional properties:
 - mac-address: mac address to be assigned to the device. Can be overridden
   by UEFI.
 - dma-coherent: Present if dma operations are coherent
+- amd,per-channel-interrupt: Indicates that Rx and Tx complete will generate
+  a unique interrupt for each DMA channel - this requires an additional
+  interrupt be configured for each DMA channel
 
 Example:
        xgbe@e0700000 {
@@ -30,7 +36,9 @@ Example:
                reg = <0 0xe0700000 0 0x80000>,
                      <0 0xe0780000 0 0x80000>;
                interrupt-parent = <&gic>;
-               interrupts = <0 325 4>;
+               interrupts = <0 325 4>,
+                            <0 326 1>, <0 327 1>, <0 328 1>, <0 329 1>;
+               amd,per-channel-interrupt;
                clocks = <&xgbe_dma_clk>, <&xgbe_ptp_clk>;
                clock-names = "dma_clk", "ptp_clk";
                phy-handle = <&phy>;
index b3719f154637a034da6e295200412a46e64a6acd..ac3d319ffab31eb397cf3f5a06ed0c0ff3688f5a 100644 (file)
@@ -481,17 +481,21 @@ static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata)
 
                if (channel->tx_ring) {
                        /* Enable the following Tx interrupts
-                        *   TIE  - Transmit Interrupt Enable (unless polling)
+                        *   TIE  - Transmit Interrupt Enable (unless using
+                        *          per channel interrupts)
                         */
-                       XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1);
+                       if (!pdata->per_channel_irq)
+                               XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1);
                }
                if (channel->rx_ring) {
                        /* Enable following Rx interrupts
                         *   RBUE - Receive Buffer Unavailable Enable
-                        *   RIE  - Receive Interrupt Enable
+                        *   RIE  - Receive Interrupt Enable (unless using
+                        *          per channel interrupts)
                         */
                        XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 1);
-                       XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1);
+                       if (!pdata->per_channel_irq)
+                               XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1);
                }
 
                XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, dma_ch_ier);
index 07e2d216323a7d13806ef612c521fef94e1d9e0a..c3533e104c61eda8f23981707dd765a3bd5fcc04 100644 (file)
  *     THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/tcp.h>
 #include <linux/if_vlan.h>
 #include "xgbe.h"
 #include "xgbe-common.h"
 
-static int xgbe_poll(struct napi_struct *, int);
+static int xgbe_one_poll(struct napi_struct *, int);
+static int xgbe_all_poll(struct napi_struct *, int);
 static void xgbe_set_rx_mode(struct net_device *);
 
 static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
@@ -134,6 +136,7 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
        struct xgbe_channel *channel_mem, *channel;
        struct xgbe_ring *tx_ring, *rx_ring;
        unsigned int count, i;
+       int ret = -ENOMEM;
 
        count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count);
 
@@ -158,6 +161,19 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
                channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE +
                                    (DMA_CH_INC * i);
 
+               if (pdata->per_channel_irq) {
+                       /* Get the DMA interrupt (offset 1) */
+                       ret = platform_get_irq(pdata->pdev, i + 1);
+                       if (ret < 0) {
+                               netdev_err(pdata->netdev,
+                                          "platform_get_irq %u failed\n",
+                                          i + 1);
+                               goto err_irq;
+                       }
+
+                       channel->dma_irq = ret;
+               }
+
                if (i < pdata->tx_ring_count) {
                        spin_lock_init(&tx_ring->lock);
                        channel->tx_ring = tx_ring++;
@@ -168,9 +184,9 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
                        channel->rx_ring = rx_ring++;
                }
 
-               DBGPR("  %s - queue_index=%u, dma_regs=%p, tx=%p, rx=%p\n",
+               DBGPR("  %s: queue=%u, dma_regs=%p, dma_irq=%d, tx=%p, rx=%p\n",
                      channel->name, channel->queue_index, channel->dma_regs,
-                     channel->tx_ring, channel->rx_ring);
+                     channel->dma_irq, channel->tx_ring, channel->rx_ring);
        }
 
        pdata->channel = channel_mem;
@@ -178,6 +194,9 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
 
        return 0;
 
+err_irq:
+       kfree(rx_ring);
+
 err_rx_ring:
        kfree(tx_ring);
 
@@ -185,9 +204,7 @@ err_tx_ring:
        kfree(channel_mem);
 
 err_channel:
-       netdev_err(pdata->netdev, "channel allocation failed\n");
-
-       return -ENOMEM;
+       return ret;
 }
 
 static void xgbe_free_channels(struct xgbe_prv_data *pdata)
@@ -287,11 +304,7 @@ static irqreturn_t xgbe_isr(int irq, void *data)
        if (!dma_isr)
                goto isr_done;
 
-       DBGPR("-->xgbe_isr\n");
-
        DBGPR("  DMA_ISR = %08x\n", dma_isr);
-       DBGPR("  DMA_DS0 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR0));
-       DBGPR("  DMA_DS1 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR1));
 
        for (i = 0; i < pdata->channel_count; i++) {
                if (!(dma_isr & (1 << i)))
@@ -302,6 +315,10 @@ static irqreturn_t xgbe_isr(int irq, void *data)
                dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR);
                DBGPR("  DMA_CH%u_ISR = %08x\n", i, dma_ch_isr);
 
+               /* If we get a TI or RI interrupt that means per channel DMA
+                * interrupts are not enabled, so we use the private data napi
+                * structure, not the per channel napi structure
+                */
                if (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, TI) ||
                    XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RI)) {
                        if (napi_schedule_prep(&pdata->napi)) {
@@ -344,12 +361,28 @@ static irqreturn_t xgbe_isr(int irq, void *data)
 
        DBGPR("  DMA_ISR = %08x\n", XGMAC_IOREAD(pdata, DMA_ISR));
 
-       DBGPR("<--xgbe_isr\n");
-
 isr_done:
        return IRQ_HANDLED;
 }
 
+static irqreturn_t xgbe_dma_isr(int irq, void *data)
+{
+       struct xgbe_channel *channel = data;
+
+       /* Per channel DMA interrupts are enabled, so we use the per
+        * channel napi structure and not the private data napi structure
+        */
+       if (napi_schedule_prep(&channel->napi)) {
+               /* Disable Tx and Rx interrupts */
+               disable_irq(channel->dma_irq);
+
+               /* Turn on polling */
+               __napi_schedule(&channel->napi);
+       }
+
+       return IRQ_HANDLED;
+}
+
 static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer)
 {
        struct xgbe_channel *channel = container_of(timer,
@@ -357,18 +390,24 @@ static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer)
                                                    tx_timer);
        struct xgbe_ring *ring = channel->tx_ring;
        struct xgbe_prv_data *pdata = channel->pdata;
+       struct napi_struct *napi;
        unsigned long flags;
 
        DBGPR("-->xgbe_tx_timer\n");
 
+       napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
+
        spin_lock_irqsave(&ring->lock, flags);
 
-       if (napi_schedule_prep(&pdata->napi)) {
+       if (napi_schedule_prep(napi)) {
                /* Disable Tx and Rx interrupts */
-               xgbe_disable_rx_tx_ints(pdata);
+               if (pdata->per_channel_irq)
+                       disable_irq(channel->dma_irq);
+               else
+                       xgbe_disable_rx_tx_ints(pdata);
 
                /* Turn on polling */
-               __napi_schedule(&pdata->napi);
+               __napi_schedule(napi);
        }
 
        channel->tx_timer_active = 0;
@@ -504,18 +543,46 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata)
 
 static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add)
 {
-       if (add)
-               netif_napi_add(pdata->netdev, &pdata->napi, xgbe_poll,
-                              NAPI_POLL_WEIGHT);
-       napi_enable(&pdata->napi);
+       struct xgbe_channel *channel;
+       unsigned int i;
+
+       if (pdata->per_channel_irq) {
+               channel = pdata->channel;
+               for (i = 0; i < pdata->channel_count; i++, channel++) {
+                       if (add)
+                               netif_napi_add(pdata->netdev, &channel->napi,
+                                              xgbe_one_poll, NAPI_POLL_WEIGHT);
+
+                       napi_enable(&channel->napi);
+               }
+       } else {
+               if (add)
+                       netif_napi_add(pdata->netdev, &pdata->napi,
+                                      xgbe_all_poll, NAPI_POLL_WEIGHT);
+
+               napi_enable(&pdata->napi);
+       }
 }
 
 static void xgbe_napi_disable(struct xgbe_prv_data *pdata, unsigned int del)
 {
-       napi_disable(&pdata->napi);
+       struct xgbe_channel *channel;
+       unsigned int i;
+
+       if (pdata->per_channel_irq) {
+               channel = pdata->channel;
+               for (i = 0; i < pdata->channel_count; i++, channel++) {
+                       napi_disable(&channel->napi);
 
-       if (del)
-               netif_napi_del(&pdata->napi);
+                       if (del)
+                               netif_napi_del(&channel->napi);
+               }
+       } else {
+               napi_disable(&pdata->napi);
+
+               if (del)
+                       netif_napi_del(&pdata->napi);
+       }
 }
 
 void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata)
@@ -828,7 +895,9 @@ static void xgbe_stop(struct xgbe_prv_data *pdata)
 
 static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset)
 {
+       struct xgbe_channel *channel;
        struct xgbe_hw_if *hw_if = &pdata->hw_if;
+       unsigned int i;
 
        DBGPR("-->xgbe_restart_dev\n");
 
@@ -837,7 +906,12 @@ static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset)
                return;
 
        xgbe_stop(pdata);
-       synchronize_irq(pdata->irq_number);
+       synchronize_irq(pdata->dev_irq);
+       if (pdata->per_channel_irq) {
+               channel = pdata->channel;
+               for (i = 0; i < pdata->channel_count; i++, channel++)
+                       synchronize_irq(channel->dma_irq);
+       }
 
        xgbe_free_tx_data(pdata);
        xgbe_free_rx_data(pdata);
@@ -1165,6 +1239,9 @@ static int xgbe_open(struct net_device *netdev)
        struct xgbe_prv_data *pdata = netdev_priv(netdev);
        struct xgbe_hw_if *hw_if = &pdata->hw_if;
        struct xgbe_desc_if *desc_if = &pdata->desc_if;
+       struct xgbe_channel *channel = NULL;
+       char dma_irq_name[IFNAMSIZ + 32];
+       unsigned int i = 0;
        int ret;
 
        DBGPR("-->xgbe_open\n");
@@ -1208,14 +1285,32 @@ static int xgbe_open(struct net_device *netdev)
        INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp);
 
        /* Request interrupts */
-       ret = devm_request_irq(pdata->dev, netdev->irq, xgbe_isr, 0,
+       ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0,
                               netdev->name, pdata);
        if (ret) {
                netdev_alert(netdev, "error requesting irq %d\n",
-                            pdata->irq_number);
+                            pdata->dev_irq);
                goto err_rings;
        }
-       pdata->irq_number = netdev->irq;
+
+       if (pdata->per_channel_irq) {
+               channel = pdata->channel;
+               for (i = 0; i < pdata->channel_count; i++, channel++) {
+                       snprintf(dma_irq_name, sizeof(dma_irq_name) - 1,
+                                "%s-TxRx-%u", netdev_name(netdev),
+                                channel->queue_index);
+
+                       ret = devm_request_irq(pdata->dev, channel->dma_irq,
+                                              xgbe_dma_isr, 0, dma_irq_name,
+                                              channel);
+                       if (ret) {
+                               netdev_alert(netdev,
+                                            "error requesting irq %d\n",
+                                            channel->dma_irq);
+                               goto err_irq;
+                       }
+               }
+       }
 
        ret = xgbe_start(pdata);
        if (ret)
@@ -1228,8 +1323,14 @@ static int xgbe_open(struct net_device *netdev)
 err_start:
        hw_if->exit(pdata);
 
-       devm_free_irq(pdata->dev, pdata->irq_number, pdata);
-       pdata->irq_number = 0;
+err_irq:
+       if (pdata->per_channel_irq) {
+               /* Using an unsigned int, 'i' will go to UINT_MAX and exit */
+               for (i--, channel--; i < pdata->channel_count; i--, channel--)
+                       devm_free_irq(pdata->dev, channel->dma_irq, channel);
+       }
+
+       devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
 
 err_rings:
        desc_if->free_ring_resources(pdata);
@@ -1254,6 +1355,8 @@ static int xgbe_close(struct net_device *netdev)
        struct xgbe_prv_data *pdata = netdev_priv(netdev);
        struct xgbe_hw_if *hw_if = &pdata->hw_if;
        struct xgbe_desc_if *desc_if = &pdata->desc_if;
+       struct xgbe_channel *channel;
+       unsigned int i;
 
        DBGPR("-->xgbe_close\n");
 
@@ -1269,10 +1372,12 @@ static int xgbe_close(struct net_device *netdev)
        /* Free the channel and ring structures */
        xgbe_free_channels(pdata);
 
-       /* Release the interrupt */
-       if (pdata->irq_number != 0) {
-               devm_free_irq(pdata->dev, pdata->irq_number, pdata);
-               pdata->irq_number = 0;
+       /* Release the interrupts */
+       devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
+       if (pdata->per_channel_irq) {
+               channel = pdata->channel;
+               for (i = 0; i < pdata->channel_count; i++, channel++)
+                       devm_free_irq(pdata->dev, channel->dma_irq, channel);
        }
 
        /* Disable the clocks */
@@ -1505,14 +1610,20 @@ static int xgbe_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
 static void xgbe_poll_controller(struct net_device *netdev)
 {
        struct xgbe_prv_data *pdata = netdev_priv(netdev);
+       struct xgbe_channel *channel;
+       unsigned int i;
 
        DBGPR("-->xgbe_poll_controller\n");
 
-       disable_irq(pdata->irq_number);
-
-       xgbe_isr(pdata->irq_number, pdata);
-
-       enable_irq(pdata->irq_number);
+       if (pdata->per_channel_irq) {
+               channel = pdata->channel;
+               for (i = 0; i < pdata->channel_count; i++, channel++)
+                       xgbe_dma_isr(channel->dma_irq, channel);
+       } else {
+               disable_irq(pdata->dev_irq);
+               xgbe_isr(pdata->dev_irq, pdata);
+               enable_irq(pdata->dev_irq);
+       }
 
        DBGPR("<--xgbe_poll_controller\n");
 }
@@ -1704,6 +1815,7 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
        struct xgbe_ring_data *rdata;
        struct xgbe_packet_data *packet;
        struct net_device *netdev = pdata->netdev;
+       struct napi_struct *napi;
        struct sk_buff *skb;
        struct skb_shared_hwtstamps *hwtstamps;
        unsigned int incomplete, error, context_next, context;
@@ -1717,6 +1829,8 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
        if (!ring)
                return 0;
 
+       napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
+
        rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
        packet = &ring->packet_data;
        while (packet_count < budget) {
@@ -1849,10 +1963,10 @@ read_again:
                skb->dev = netdev;
                skb->protocol = eth_type_trans(skb, netdev);
                skb_record_rx_queue(skb, channel->queue_index);
-               skb_mark_napi_id(skb, &pdata->napi);
+               skb_mark_napi_id(skb, napi);
 
                netdev->last_rx = jiffies;
-               napi_gro_receive(&pdata->napi, skb);
+               napi_gro_receive(napi, skb);
 
 next_packet:
                packet_count++;
@@ -1874,7 +1988,35 @@ next_packet:
        return packet_count;
 }
 
-static int xgbe_poll(struct napi_struct *napi, int budget)
+static int xgbe_one_poll(struct napi_struct *napi, int budget)
+{
+       struct xgbe_channel *channel = container_of(napi, struct xgbe_channel,
+                                                   napi);
+       int processed = 0;
+
+       DBGPR("-->xgbe_one_poll: budget=%d\n", budget);
+
+       /* Cleanup Tx ring first */
+       xgbe_tx_poll(channel);
+
+       /* Process Rx ring next */
+       processed = xgbe_rx_poll(channel, budget);
+
+       /* If we processed everything, we are done */
+       if (processed < budget) {
+               /* Turn off polling */
+               napi_complete(napi);
+
+               /* Enable Tx and Rx interrupts */
+               enable_irq(channel->dma_irq);
+       }
+
+       DBGPR("<--xgbe_one_poll: received = %d\n", processed);
+
+       return processed;
+}
+
+static int xgbe_all_poll(struct napi_struct *napi, int budget)
 {
        struct xgbe_prv_data *pdata = container_of(napi, struct xgbe_prv_data,
                                                   napi);
@@ -1883,7 +2025,7 @@ static int xgbe_poll(struct napi_struct *napi, int budget)
        int processed, last_processed;
        unsigned int i;
 
-       DBGPR("-->xgbe_poll: budget=%d\n", budget);
+       DBGPR("-->xgbe_all_poll: budget=%d\n", budget);
 
        processed = 0;
        ring_budget = budget / pdata->rx_ring_count;
@@ -1911,7 +2053,7 @@ static int xgbe_poll(struct napi_struct *napi, int budget)
                xgbe_enable_rx_tx_ints(pdata);
        }
 
-       DBGPR("<--xgbe_poll: received = %d\n", processed);
+       DBGPR("<--xgbe_all_poll: received = %d\n", processed);
 
        return processed;
 }
index e5077fd5b01234592b5ba636f6f2cee2c1843196..cff9902d1456dd6e51dc19fe3a5ef49685d3d729 100644 (file)
@@ -264,12 +264,18 @@ static int xgbe_probe(struct platform_device *pdev)
                pdata->awcache = XGBE_DMA_SYS_AWCACHE;
        }
 
+       /* Check for per channel interrupt support */
+       if (of_property_read_bool(dev->of_node, XGBE_DMA_IRQS))
+               pdata->per_channel_irq = 1;
+
        ret = platform_get_irq(pdev, 0);
        if (ret < 0) {
-               dev_err(dev, "platform_get_irq failed\n");
+               dev_err(dev, "platform_get_irq failed\n");
                goto err_io;
        }
-       netdev->irq = ret;
+       pdata->dev_irq = ret;
+
+       netdev->irq = pdata->dev_irq;
        netdev->base_addr = (unsigned long)pdata->xgmac_regs;
 
        /* Set all the function pointers */
index 1480c9d41821102fce9fe3d37a9b377d9368e7fb..55c935f4884a59c4cf29187664d852bb98c12d1a 100644 (file)
 /* Device-tree clock names */
 #define XGBE_DMA_CLOCK         "dma_clk"
 #define XGBE_PTP_CLOCK         "ptp_clk"
+#define XGBE_DMA_IRQS          "amd,per-channel-interrupt"
 
 /* Timestamp support - values based on 50MHz PTP clock
  *   50MHz => 20 nsec
@@ -359,6 +360,12 @@ struct xgbe_channel {
        unsigned int queue_index;
        void __iomem *dma_regs;
 
+       /* Per channel interrupt irq number */
+       int dma_irq;
+
+       /* Netdev related settings */
+       struct napi_struct napi;
+
        unsigned int saved_ier;
 
        unsigned int tx_timer_active;
@@ -609,7 +616,8 @@ struct xgbe_prv_data {
        /* XPCS indirect addressing mutex */
        struct mutex xpcs_mutex;
 
-       int irq_number;
+       int dev_irq;
+       unsigned int per_channel_irq;
 
        struct xgbe_hw_if hw_if;
        struct xgbe_desc_if desc_if;