diff --git a/drivers/net/ixgbe/ixgbe.h b/drivers/net/ixgbe/ixgbe.h index 20774772b608..79f5519e2aa9 100644 --- a/drivers/net/ixgbe/ixgbe.h +++ b/drivers/net/ixgbe/ixgbe.h @@ -136,6 +136,8 @@ struct ixgbe_ring { u16 head; u16 tail; + unsigned int total_bytes; + unsigned int total_packets; u16 reg_idx; /* holds the special value that gets the hardware register * offset associated with this ring, which is different @@ -174,6 +176,8 @@ struct ixgbe_q_vector { DECLARE_BITMAP(txr_idx, MAX_TX_QUEUES); /* Tx ring indices */ u8 rxr_count; /* Rx ring count assigned to this vector */ u8 txr_count; /* Tx ring count assigned to this vector */ + u8 tx_eitr; + u8 rx_eitr; u32 eitr; }; @@ -215,6 +219,11 @@ struct ixgbe_adapter { struct ixgbe_q_vector q_vector[MAX_MSIX_Q_VECTORS]; char name[MAX_MSIX_COUNT][IFNAMSIZ + 5]; + /* Interrupt Throttle Rate */ + u32 itr_setting; + u16 eitr_low; + u16 eitr_high; + /* TX */ struct ixgbe_ring *tx_ring; /* One per active queue */ u64 restart_queue; diff --git a/drivers/net/ixgbe/ixgbe_ethtool.c b/drivers/net/ixgbe/ixgbe_ethtool.c index 85b7d15e1217..4e463778bcfd 100644 --- a/drivers/net/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ixgbe/ixgbe_ethtool.c @@ -886,13 +886,13 @@ static int ixgbe_get_coalesce(struct net_device *netdev, { struct ixgbe_adapter *adapter = netdev_priv(netdev); - if (adapter->rx_eitr == 0) - ec->rx_coalesce_usecs = 0; + if (adapter->rx_eitr < IXGBE_MIN_ITR_USECS) + ec->rx_coalesce_usecs = adapter->rx_eitr; else ec->rx_coalesce_usecs = 1000000 / adapter->rx_eitr; - if (adapter->tx_eitr == 0) - ec->tx_coalesce_usecs = 0; + if (adapter->tx_eitr < IXGBE_MIN_ITR_USECS) + ec->tx_coalesce_usecs = adapter->tx_eitr; else ec->tx_coalesce_usecs = 1000000 / adapter->tx_eitr; @@ -906,22 +906,26 @@ static int ixgbe_set_coalesce(struct net_device *netdev, struct ixgbe_adapter *adapter = netdev_priv(netdev); if ((ec->rx_coalesce_usecs > IXGBE_MAX_ITR_USECS) || - ((ec->rx_coalesce_usecs > 0) && + ((ec->rx_coalesce_usecs != 0) && + (ec->rx_coalesce_usecs != 1) && + (ec->rx_coalesce_usecs != 3) && (ec->rx_coalesce_usecs < IXGBE_MIN_ITR_USECS))) return -EINVAL; if ((ec->tx_coalesce_usecs > IXGBE_MAX_ITR_USECS) || - ((ec->tx_coalesce_usecs > 0) && + ((ec->tx_coalesce_usecs != 0) && + (ec->tx_coalesce_usecs != 1) && + (ec->tx_coalesce_usecs != 3) && (ec->tx_coalesce_usecs < IXGBE_MIN_ITR_USECS))) return -EINVAL; /* convert to rate of irq's per second */ - if (ec->rx_coalesce_usecs == 0) - adapter->rx_eitr = 0; + if (ec->rx_coalesce_usecs < IXGBE_MIN_ITR_USECS) + adapter->rx_eitr = ec->rx_coalesce_usecs; else adapter->rx_eitr = (1000000 / ec->rx_coalesce_usecs); - if (ec->tx_coalesce_usecs == 0) - adapter->tx_eitr = 0; + if (ec->tx_coalesce_usecs < IXGBE_MIN_ITR_USECS) + adapter->tx_eitr = ec->rx_coalesce_usecs; else adapter->tx_eitr = (1000000 / ec->tx_coalesce_usecs); diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c index fc1c226f5751..e1f11e686f47 100644 --- a/drivers/net/ixgbe/ixgbe_main.c +++ b/drivers/net/ixgbe/ixgbe_main.c @@ -282,6 +282,8 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_adapter *adapter, if (total_tx_packets >= tx_ring->work_limit) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, tx_ring->eims_value); + tx_ring->total_bytes += total_tx_bytes; + tx_ring->total_packets += total_tx_packets; adapter->net_stats.tx_bytes += total_tx_bytes; adapter->net_stats.tx_packets += total_tx_packets; cleaned = total_tx_packets ? true : false; @@ -568,6 +570,11 @@ next_desc: adapter->net_stats.rx_bytes += total_rx_bytes; adapter->net_stats.rx_packets += total_rx_packets; + rx_ring->total_packets += total_rx_packets; + rx_ring->total_bytes += total_rx_bytes; + adapter->net_stats.rx_bytes += total_rx_bytes; + adapter->net_stats.rx_packets += total_rx_packets; + return cleaned; } @@ -634,6 +641,144 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); } +enum latency_range { + lowest_latency = 0, + low_latency = 1, + bulk_latency = 2, + latency_invalid = 255 +}; + +/** + * ixgbe_update_itr - update the dynamic ITR value based on statistics + * @adapter: pointer to adapter + * @eitr: eitr setting (ints per sec) to give last timeslice + * @itr_setting: current throttle rate in ints/second + * @packets: the number of packets during this measurement interval + * @bytes: the number of bytes during this measurement interval + * + * Stores a new ITR value based on packets and byte + * counts during the last interrupt. The advantage of per interrupt + * computation is faster updates and more accurate ITR for the current + * traffic pattern. Constants in this function were computed + * based on theoretical maximum wire speed and thresholds were set based + * on testing data as well as attempting to minimize response time + * while increasing bulk throughput. + * this functionality is controlled by the InterruptThrottleRate module + * parameter (see ixgbe_param.c) + **/ +static u8 ixgbe_update_itr(struct ixgbe_adapter *adapter, + u32 eitr, u8 itr_setting, + int packets, int bytes) +{ + unsigned int retval = itr_setting; + u32 timepassed_us; + u64 bytes_perint; + + if (packets == 0) + goto update_itr_done; + + + /* simple throttlerate management + * 0-20MB/s lowest (100000 ints/s) + * 20-100MB/s low (20000 ints/s) + * 100-1249MB/s bulk (8000 ints/s) + */ + /* what was last interrupt timeslice? */ + timepassed_us = 1000000/eitr; + bytes_perint = bytes / timepassed_us; /* bytes/usec */ + + switch (itr_setting) { + case lowest_latency: + if (bytes_perint > adapter->eitr_low) + retval = low_latency; + break; + case low_latency: + if (bytes_perint > adapter->eitr_high) + retval = bulk_latency; + else if (bytes_perint <= adapter->eitr_low) + retval = lowest_latency; + break; + case bulk_latency: + if (bytes_perint <= adapter->eitr_high) + retval = low_latency; + break; + } + +update_itr_done: + return retval; +} + +static void ixgbe_set_itr_msix(struct ixgbe_q_vector *q_vector) +{ + struct ixgbe_adapter *adapter = q_vector->adapter; + struct ixgbe_hw *hw = &adapter->hw; + u32 new_itr; + u8 current_itr, ret_itr; + int i, r_idx, v_idx = ((void *)q_vector - (void *)(adapter->q_vector)) / + sizeof(struct ixgbe_q_vector); + struct ixgbe_ring *rx_ring, *tx_ring; + + r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); + for (i = 0; i < q_vector->txr_count; i++) { + tx_ring = &(adapter->tx_ring[r_idx]); + ret_itr = ixgbe_update_itr(adapter, q_vector->eitr, + q_vector->tx_eitr, + tx_ring->total_packets, + tx_ring->total_bytes); + /* if the result for this queue would decrease interrupt + * rate for this vector then use that result */ + q_vector->tx_eitr = ((q_vector->tx_eitr > ret_itr) ? + q_vector->tx_eitr - 1 : ret_itr); + r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, + r_idx + 1); + } + + r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); + for (i = 0; i < q_vector->rxr_count; i++) { + rx_ring = &(adapter->rx_ring[r_idx]); + ret_itr = ixgbe_update_itr(adapter, q_vector->eitr, + q_vector->rx_eitr, + rx_ring->total_packets, + rx_ring->total_bytes); + /* if the result for this queue would decrease interrupt + * rate for this vector then use that result */ + q_vector->rx_eitr = ((q_vector->rx_eitr > ret_itr) ? + q_vector->rx_eitr - 1 : ret_itr); + r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, + r_idx + 1); + } + + current_itr = max(q_vector->rx_eitr, q_vector->tx_eitr); + + switch (current_itr) { + /* counts and packets in update_itr are dependent on these numbers */ + case lowest_latency: + new_itr = 100000; + break; + case low_latency: + new_itr = 20000; /* aka hwitr = ~200 */ + break; + case bulk_latency: + default: + new_itr = 8000; + break; + } + + if (new_itr != q_vector->eitr) { + u32 itr_reg; + /* do an exponential smoothing */ + new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100); + q_vector->eitr = new_itr; + itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr); + /* must write high and low 16 bits to reset counter */ + DPRINTK(TX_ERR, DEBUG, "writing eitr(%d): %08X\n", v_idx, + itr_reg); + IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg | (itr_reg)<<16); + } + + return; +} + static irqreturn_t ixgbe_msix_lsc(int irq, void *data) { struct net_device *netdev = data; @@ -666,12 +811,13 @@ static irqreturn_t ixgbe_msix_clean_tx(int irq, void *data) r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); for (i = 0; i < q_vector->txr_count; i++) { txr = &(adapter->tx_ring[r_idx]); + txr->total_bytes = 0; + txr->total_packets = 0; ixgbe_clean_tx_irq(adapter, txr); r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, r_idx + 1); } - return IRQ_HANDLED; } @@ -694,6 +840,8 @@ static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data) rxr = &(adapter->rx_ring[r_idx]); /* disable interrupts on this vector only */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, rxr->v_idx); + rxr->total_bytes = 0; + rxr->total_packets = 0; netif_rx_schedule(adapter->netdev, &q_vector->napi); return IRQ_HANDLED; @@ -730,6 +878,8 @@ static int ixgbe_clean_rxonly(struct napi_struct *napi, int budget) /* If all Rx work done, exit the polling mode */ if (work_done < budget) { netif_rx_complete(adapter->netdev, napi); + if (adapter->rx_eitr < IXGBE_MIN_ITR_USECS) + ixgbe_set_itr_msix(q_vector); if (!test_bit(__IXGBE_DOWN, &adapter->state)) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->v_idx); } @@ -886,6 +1036,54 @@ out: return err; } +static void ixgbe_set_itr(struct ixgbe_adapter *adapter) +{ + struct ixgbe_hw *hw = &adapter->hw; + struct ixgbe_q_vector *q_vector = adapter->q_vector; + u8 current_itr; + u32 new_itr = q_vector->eitr; + struct ixgbe_ring *rx_ring = &adapter->rx_ring[0]; + struct ixgbe_ring *tx_ring = &adapter->tx_ring[0]; + + q_vector->tx_eitr = ixgbe_update_itr(adapter, new_itr, + q_vector->tx_eitr, + tx_ring->total_packets, + tx_ring->total_bytes); + q_vector->rx_eitr = ixgbe_update_itr(adapter, new_itr, + q_vector->rx_eitr, + rx_ring->total_packets, + rx_ring->total_bytes); + + current_itr = max(q_vector->rx_eitr, q_vector->tx_eitr); + + switch (current_itr) { + /* counts and packets in update_itr are dependent on these numbers */ + case lowest_latency: + new_itr = 100000; + break; + case low_latency: + new_itr = 20000; /* aka hwitr = ~200 */ + break; + case bulk_latency: + new_itr = 8000; + break; + default: + break; + } + + if (new_itr != q_vector->eitr) { + u32 itr_reg; + /* do an exponential smoothing */ + new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100); + q_vector->eitr = new_itr; + itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr); + /* must write high and low 16 bits to reset counter */ + IXGBE_WRITE_REG(hw, IXGBE_EITR(0), itr_reg | (itr_reg)<<16); + } + + return; +} + static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter); /** @@ -916,6 +1114,10 @@ static irqreturn_t ixgbe_intr(int irq, void *data) if (netif_rx_schedule_prep(netdev, &adapter->q_vector[0].napi)) { + adapter->tx_ring[0].total_packets = 0; + adapter->tx_ring[0].total_bytes = 0; + adapter->rx_ring[0].total_packets = 0; + adapter->rx_ring[0].total_bytes = 0; /* would disable interrupts here but EIAM disabled it */ __netif_rx_schedule(netdev, &adapter->q_vector[0].napi); } @@ -1367,7 +1569,6 @@ static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter) } } - static void ixgbe_configure(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; @@ -1732,6 +1933,8 @@ static int ixgbe_poll(struct napi_struct *napi, int budget) /* If budget not fully consumed, exit the polling mode */ if (work_done < budget) { netif_rx_complete(adapter->netdev, napi); + if (adapter->rx_eitr < IXGBE_MIN_ITR_USECS) + ixgbe_set_itr(adapter); if (!test_bit(__IXGBE_DOWN, &adapter->state)) ixgbe_irq_enable(adapter); } @@ -2088,6 +2291,10 @@ static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter) adapter->ring_feature[RING_F_RSS].indices = rss; adapter->flags |= IXGBE_FLAG_RSS_ENABLED; + /* Enable Dynamic interrupt throttling by default */ + adapter->rx_eitr = 1; + adapter->tx_eitr = 1; + /* default flow control settings */ hw->fc.original_type = ixgbe_fc_full; hw->fc.type = ixgbe_fc_full;