sfc: Implement adaptive IRQ moderation

Calculate a score for each 1000 IRQs:
- TX completions are worth 1 point
- RX completions are worth 4 if merged using LRO or 2 otherwise

Reduce moderation if the score is less than 10000, down to a minimum
of 5 us.  Increase moderation if the score is more than 20000, up to
the specified maximum.

Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/sfc/efx.c b/drivers/net/sfc/efx.c
index 8fa68d82c022d505df29b95217f0368ac14cec60..6eff9ca6c6c81c87df26b418f6c31cb3a94fa908 100644 (file)
--- a/drivers/net/sfc/efx.c
+++ b/drivers/net/sfc/efx.c
@@ -133,6 +133,16 @@ static int phy_flash_cfg;
 module_param(phy_flash_cfg, int, 0644);
 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
 
+static unsigned irq_adapt_low_thresh = 10000;
+module_param(irq_adapt_low_thresh, uint, 0644);
+MODULE_PARM_DESC(irq_adapt_low_thresh,
+                "Threshold score for reducing IRQ moderation");
+
+static unsigned irq_adapt_high_thresh = 20000;
+module_param(irq_adapt_high_thresh, uint, 0644);
+MODULE_PARM_DESC(irq_adapt_high_thresh,
+                "Threshold score for increasing IRQ moderation");
+
 /**************************************************************************
  *
  * Utility functions and prototypes
@@ -223,6 +233,35 @@ static int efx_poll(struct napi_struct *napi, int budget)
        rx_packets = efx_process_channel(channel, budget);
 
        if (rx_packets < budget) {
+               struct efx_nic *efx = channel->efx;
+
+               if (channel->used_flags & EFX_USED_BY_RX &&
+                   efx->irq_rx_adaptive &&
+                   unlikely(++channel->irq_count == 1000)) {
+                       unsigned old_irq_moderation = channel->irq_moderation;
+
+                       if (unlikely(channel->irq_mod_score <
+                                    irq_adapt_low_thresh)) {
+                               channel->irq_moderation =
+                                       max_t(int,
+                                             channel->irq_moderation -
+                                             FALCON_IRQ_MOD_RESOLUTION,
+                                             FALCON_IRQ_MOD_RESOLUTION);
+                       } else if (unlikely(channel->irq_mod_score >
+                                           irq_adapt_high_thresh)) {
+                               channel->irq_moderation =
+                                       min(channel->irq_moderation +
+                                           FALCON_IRQ_MOD_RESOLUTION,
+                                           efx->irq_rx_moderation);
+                       }
+
+                       if (channel->irq_moderation != old_irq_moderation)
+                               falcon_set_int_moderation(channel);
+
+                       channel->irq_count = 0;
+                       channel->irq_mod_score = 0;
+               }
+
                /* There is no race here; although napi_disable() will
                 * only wait for napi_complete(), this isn't a problem
                 * since efx_channel_processed() will have no effect if
@@ -991,7 +1030,7 @@ static int efx_probe_nic(struct efx_nic *efx)
        efx_set_channels(efx);
 
        /* Initialise the interrupt moderation settings */
-       efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec);
+       efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true);
 
        return 0;
 }
@@ -1188,7 +1227,8 @@ void efx_flush_queues(struct efx_nic *efx)
  **************************************************************************/
 
 /* Set interrupt moderation parameters */
-void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs)
+void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,
+                            bool rx_adaptive)
 {
        struct efx_tx_queue *tx_queue;
        struct efx_rx_queue *rx_queue;
@@ -1198,6 +1238,8 @@ void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs)
        efx_for_each_tx_queue(tx_queue, efx)
                tx_queue->channel->irq_moderation = tx_usecs;
 
+       efx->irq_rx_adaptive = rx_adaptive;
+       efx->irq_rx_moderation = rx_usecs;
        efx_for_each_rx_queue(rx_queue, efx)
                rx_queue->channel->irq_moderation = rx_usecs;
 }

diff --git a/drivers/net/sfc/efx.h b/drivers/net/sfc/efx.h
index 8bde1d2a21db3f7108d3126e3edf9aeb76a07f3c..da157aa74b836cb5527158ef75088872aacf9873 100644 (file)
--- a/drivers/net/sfc/efx.h
+++ b/drivers/net/sfc/efx.h
@@ -52,7 +52,7 @@ extern void efx_schedule_reset(struct efx_nic *efx, enum reset_type type);
 extern void efx_suspend(struct efx_nic *efx);
 extern void efx_resume(struct efx_nic *efx);
 extern void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs,
-                                   int rx_usecs);
+                                   int rx_usecs, bool rx_adaptive);
 extern int efx_request_power(struct efx_nic *efx, int mw, const char *name);
 extern void efx_hex_dump(const u8 *, unsigned int, const char *);
 

diff --git a/drivers/net/sfc/ethtool.c b/drivers/net/sfc/ethtool.c
index 589d1329296913e8ac38b3d3cf1ae1dffd95704b..64309f4e8b190e440307a4da29b47753cf794416 100644 (file)
--- a/drivers/net/sfc/ethtool.c
+++ b/drivers/net/sfc/ethtool.c
@@ -604,7 +604,6 @@ static int efx_ethtool_get_coalesce(struct net_device *net_dev,
 {
        struct efx_nic *efx = netdev_priv(net_dev);
        struct efx_tx_queue *tx_queue;
-       struct efx_rx_queue *rx_queue;
        struct efx_channel *channel;
 
        memset(coalesce, 0, sizeof(*coalesce));
@@ -622,14 +621,8 @@ static int efx_ethtool_get_coalesce(struct net_device *net_dev,
                }
        }
 
-       /* Find lowest IRQ moderation across all used RX queues */
-       coalesce->rx_coalesce_usecs_irq = ~((u32) 0);
-       efx_for_each_rx_queue(rx_queue, efx) {
-               channel = rx_queue->channel;
-               if (channel->irq_moderation < coalesce->rx_coalesce_usecs_irq)
-                       coalesce->rx_coalesce_usecs_irq =
-                               channel->irq_moderation;
-       }
+       coalesce->use_adaptive_rx_coalesce = efx->irq_rx_adaptive;
+       coalesce->rx_coalesce_usecs_irq = efx->irq_rx_moderation;
 
        return 0;
 }
@@ -643,10 +636,9 @@ static int efx_ethtool_set_coalesce(struct net_device *net_dev,
        struct efx_nic *efx = netdev_priv(net_dev);
        struct efx_channel *channel;
        struct efx_tx_queue *tx_queue;
-       unsigned tx_usecs, rx_usecs;
+       unsigned tx_usecs, rx_usecs, adaptive;
 
-       if (coalesce->use_adaptive_rx_coalesce ||
-           coalesce->use_adaptive_tx_coalesce)
+       if (coalesce->use_adaptive_tx_coalesce)
                return -EOPNOTSUPP;
 
        if (coalesce->rx_coalesce_usecs || coalesce->tx_coalesce_usecs) {
@@ -657,6 +649,7 @@ static int efx_ethtool_set_coalesce(struct net_device *net_dev,
 
        rx_usecs = coalesce->rx_coalesce_usecs_irq;
        tx_usecs = coalesce->tx_coalesce_usecs_irq;
+       adaptive = coalesce->use_adaptive_rx_coalesce;
 
        /* If the channel is shared only allow RX parameters to be set */
        efx_for_each_tx_queue(tx_queue, efx) {
@@ -668,7 +661,7 @@ static int efx_ethtool_set_coalesce(struct net_device *net_dev,
                }
        }
 
-       efx_init_irq_moderation(efx, tx_usecs, rx_usecs);
+       efx_init_irq_moderation(efx, tx_usecs, rx_usecs, adaptive);
 
        /* Reset channel to pick up new moderation value.  Note that
         * this may change the value of the irq_moderation field

diff --git a/drivers/net/sfc/falcon.c b/drivers/net/sfc/falcon.c
index f42fc60d1df4a7cab84130e09dc0542488f1e1f7..23a1b148d5b236347531e42a030bdbd912d1d3c5 100644 (file)
--- a/drivers/net/sfc/falcon.c
+++ b/drivers/net/sfc/falcon.c
@@ -729,6 +729,9 @@ static void falcon_handle_tx_event(struct efx_channel *channel,
                tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, TX_EV_DESC_PTR);
                tx_ev_q_label = EFX_QWORD_FIELD(*event, TX_EV_Q_LABEL);
                tx_queue = &efx->tx_queue[tx_ev_q_label];
+               channel->irq_mod_score +=
+                       (tx_ev_desc_ptr - tx_queue->read_count) &
+                       efx->type->txd_ring_mask;
                efx_xmit_done(tx_queue, tx_ev_desc_ptr);
        } else if (EFX_QWORD_FIELD(*event, TX_EV_WQ_FF_FULL)) {
                /* Rewrite the FIFO write pointer */
@@ -898,6 +901,8 @@ static void falcon_handle_rx_event(struct efx_channel *channel,
                        discard = true;
        }
 
+       channel->irq_mod_score += 2;
+
        /* Handle received packet */
        efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt,
                      checksummed, discard);
@@ -1075,14 +1080,15 @@ void falcon_set_int_moderation(struct efx_channel *channel)
                 * program is based at 0.  So actual interrupt moderation
                 * achieved is ((x + 1) * res).
                 */
-               unsigned int res = 5;
-               channel->irq_moderation -= (channel->irq_moderation % res);
-               if (channel->irq_moderation < res)
-                       channel->irq_moderation = res;
+               channel->irq_moderation -= (channel->irq_moderation %
+                                           FALCON_IRQ_MOD_RESOLUTION);
+               if (channel->irq_moderation < FALCON_IRQ_MOD_RESOLUTION)
+                       channel->irq_moderation = FALCON_IRQ_MOD_RESOLUTION;
                EFX_POPULATE_DWORD_2(timer_cmd,
                                     TIMER_MODE, TIMER_MODE_INT_HLDOFF,
                                     TIMER_VAL,
-                                    (channel->irq_moderation / res) - 1);
+                                    channel->irq_moderation /
+                                    FALCON_IRQ_MOD_RESOLUTION - 1);
        } else {
                EFX_POPULATE_DWORD_2(timer_cmd,
                                     TIMER_MODE, TIMER_MODE_DIS,

diff --git a/drivers/net/sfc/falcon.h b/drivers/net/sfc/falcon.h
index 7869c3d74383cc971e818bda22ec5fc78cbf2395..77f2e0db7ca109e8d8d17760ea322b0978fd7c4f 100644 (file)
--- a/drivers/net/sfc/falcon.h
+++ b/drivers/net/sfc/falcon.h
@@ -85,6 +85,8 @@ extern void falcon_set_int_moderation(struct efx_channel *channel);
 extern void falcon_disable_interrupts(struct efx_nic *efx);
 extern void falcon_fini_interrupt(struct efx_nic *efx);
 
+#define FALCON_IRQ_MOD_RESOLUTION 5
+
 /* Global Resources */
 extern int falcon_probe_nic(struct efx_nic *efx);
 extern int falcon_probe_resources(struct efx_nic *efx);

diff --git a/drivers/net/sfc/net_driver.h b/drivers/net/sfc/net_driver.h
index b81fc727dfffa225e95e79f0bc9ef993983e7824..e169e5dcd1e63cc2f0f39cfb11d2975271b2f8f3 100644 (file)
--- a/drivers/net/sfc/net_driver.h
+++ b/drivers/net/sfc/net_driver.h
@@ -336,6 +336,8 @@ enum efx_rx_alloc_method {
  * @eventq_read_ptr: Event queue read pointer
  * @last_eventq_read_ptr: Last event queue read pointer value.
  * @eventq_magic: Event queue magic value for driver-generated test events
+ * @irq_count: Number of IRQs since last adaptive moderation decision
+ * @irq_mod_score: IRQ moderation score
  * @rx_alloc_level: Watermark based heuristic counter for pushing descriptors
  *     and diagnostic counters
  * @rx_alloc_push_pages: RX allocation method currently in use for pushing
@@ -364,6 +366,9 @@ struct efx_channel {
        unsigned int last_eventq_read_ptr;
        unsigned int eventq_magic;
 
+       unsigned int irq_count;
+       unsigned int irq_mod_score;
+
        int rx_alloc_level;
        int rx_alloc_push_pages;
 
@@ -703,6 +708,8 @@ union efx_multicast_hash {
  * @membase: Memory BAR value
  * @biu_lock: BIU (bus interface unit) lock
  * @interrupt_mode: Interrupt mode
+ * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
+ * @irq_rx_moderation: IRQ moderation time for RX event queues
  * @i2c_adap: I2C adapter
  * @board_info: Board-level information
  * @state: Device state flag. Serialised by the rtnl_lock.
@@ -784,6 +791,8 @@ struct efx_nic {
        void __iomem *membase;
        spinlock_t biu_lock;
        enum efx_int_mode interrupt_mode;
+       bool irq_rx_adaptive;
+       unsigned int irq_rx_moderation;
 
        struct i2c_adapter i2c_adap;
        struct efx_board board_info;