net: move rx and tx hash functions to net/core/flow_dissector.c

__skb_tx_hash() and __skb_get_rxhash() are all for calculating hash
value based by some fields in skb, mostly used for selecting queues
by device drivers.

Meanwhile, net/core/dev.c is bloating.

Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: Cong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/core/dev.c b/net/core/dev.c
index b6d2b32933ba13210ab8601d974a9cb585745302..c69cd8721b284051ee29febdb3f778ba1df2935a 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -134,7 +134,6 @@
 #include <linux/cpu_rmap.h>
 #include <linux/net_tstamp.h>
 #include <linux/static_key.h>
-#include <net/flow_keys.h>
 
 #include "net-sysfs.h"
 
@@ -2636,136 +2635,6 @@ out:
        return rc;
 }
 
-static u32 hashrnd __read_mostly;
-
-/*
- * Returns a Tx hash based on the given packet descriptor a Tx queues' number
- * to be used as a distribution range.
- */
-u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
-                 unsigned int num_tx_queues)
-{
-       u32 hash;
-       u16 qoffset = 0;
-       u16 qcount = num_tx_queues;
-
-       if (skb_rx_queue_recorded(skb)) {
-               hash = skb_get_rx_queue(skb);
-               while (unlikely(hash >= num_tx_queues))
-                       hash -= num_tx_queues;
-               return hash;
-       }
-
-       if (dev->num_tc) {
-               u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
-               qoffset = dev->tc_to_txq[tc].offset;
-               qcount = dev->tc_to_txq[tc].count;
-       }
-
-       if (skb->sk && skb->sk->sk_hash)
-               hash = skb->sk->sk_hash;
-       else
-               hash = (__force u16) skb->protocol;
-       hash = jhash_1word(hash, hashrnd);
-
-       return (u16) (((u64) hash * qcount) >> 32) + qoffset;
-}
-EXPORT_SYMBOL(__skb_tx_hash);
-
-static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
-{
-       if (unlikely(queue_index >= dev->real_num_tx_queues)) {
-               net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
-                                    dev->name, queue_index,
-                                    dev->real_num_tx_queues);
-               return 0;
-       }
-       return queue_index;
-}
-
-static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
-{
-#ifdef CONFIG_XPS
-       struct xps_dev_maps *dev_maps;
-       struct xps_map *map;
-       int queue_index = -1;
-
-       rcu_read_lock();
-       dev_maps = rcu_dereference(dev->xps_maps);
-       if (dev_maps) {
-               map = rcu_dereference(
-                   dev_maps->cpu_map[raw_smp_processor_id()]);
-               if (map) {
-                       if (map->len == 1)
-                               queue_index = map->queues[0];
-                       else {
-                               u32 hash;
-                               if (skb->sk && skb->sk->sk_hash)
-                                       hash = skb->sk->sk_hash;
-                               else
-                                       hash = (__force u16) skb->protocol ^
-                                           skb->rxhash;
-                               hash = jhash_1word(hash, hashrnd);
-                               queue_index = map->queues[
-                                   ((u64)hash * map->len) >> 32];
-                       }
-                       if (unlikely(queue_index >= dev->real_num_tx_queues))
-                               queue_index = -1;
-               }
-       }
-       rcu_read_unlock();
-
-       return queue_index;
-#else
-       return -1;
-#endif
-}
-
-u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
-{
-       struct sock *sk = skb->sk;
-       int queue_index = sk_tx_queue_get(sk);
-
-       if (queue_index < 0 || skb->ooo_okay ||
-           queue_index >= dev->real_num_tx_queues) {
-               int new_index = get_xps_queue(dev, skb);
-               if (new_index < 0)
-                       new_index = skb_tx_hash(dev, skb);
-
-               if (queue_index != new_index && sk) {
-                       struct dst_entry *dst =
-                                   rcu_dereference_check(sk->sk_dst_cache, 1);
-
-                       if (dst && skb_dst(skb) == dst)
-                               sk_tx_queue_set(sk, queue_index);
-
-               }
-
-               queue_index = new_index;
-       }
-
-       return queue_index;
-}
-EXPORT_SYMBOL(__netdev_pick_tx);
-
-struct netdev_queue *netdev_pick_tx(struct net_device *dev,
-                                   struct sk_buff *skb)
-{
-       int queue_index = 0;
-
-       if (dev->real_num_tx_queues != 1) {
-               const struct net_device_ops *ops = dev->netdev_ops;
-               if (ops->ndo_select_queue)
-                       queue_index = ops->ndo_select_queue(dev, skb);
-               else
-                       queue_index = __netdev_pick_tx(dev, skb);
-               queue_index = dev_cap_txqueue(dev, queue_index);
-       }
-
-       skb_set_queue_mapping(skb, queue_index);
-       return netdev_get_tx_queue(dev, queue_index);
-}
-
 static void qdisc_pkt_len_init(struct sk_buff *skb)
 {
        const struct skb_shared_info *shinfo = skb_shinfo(skb);
@@ -3015,41 +2884,6 @@ static inline void ____napi_schedule(struct softnet_data *sd,
        __raise_softirq_irqoff(NET_RX_SOFTIRQ);
 }
 
-/*
- * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
- * and src/dst port numbers.  Sets rxhash in skb to non-zero hash value
- * on success, zero indicates no valid hash.  Also, sets l4_rxhash in skb
- * if hash is a canonical 4-tuple hash over transport ports.
- */
-void __skb_get_rxhash(struct sk_buff *skb)
-{
-       struct flow_keys keys;
-       u32 hash;
-
-       if (!skb_flow_dissect(skb, &keys))
-               return;
-
-       if (keys.ports)
-               skb->l4_rxhash = 1;
-
-       /* get a consistent hash (same value on both flow directions) */
-       if (((__force u32)keys.dst < (__force u32)keys.src) ||
-           (((__force u32)keys.dst == (__force u32)keys.src) &&
-            ((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) {
-               swap(keys.dst, keys.src);
-               swap(keys.port16[0], keys.port16[1]);
-       }
-
-       hash = jhash_3words((__force u32)keys.dst,
-                           (__force u32)keys.src,
-                           (__force u32)keys.ports, hashrnd);
-       if (!hash)
-               hash = 1;
-
-       skb->rxhash = hash;
-}
-EXPORT_SYMBOL(__skb_get_rxhash);
-
 #ifdef CONFIG_RPS
 
 /* One global table that all flow-based protocols share. */
@@ -7308,12 +7142,3 @@ out:
 }
 
 subsys_initcall(net_dev_init);
-
-static int __init initialize_hashrnd(void)
-{
-       get_random_bytes(&hashrnd, sizeof(hashrnd));
-       return 0;
-}
-
-late_initcall_sync(initialize_hashrnd);
-

diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
index 466820b6e34400787a18fb0be5f26845c2fc0629..9d4c7201400d154ab2fd00ccfab4f22f8bddb5ff 100644 (file)
--- a/net/core/flow_dissector.c
+++ b/net/core/flow_dissector.c
@@ -143,3 +143,176 @@ ipv6:
        return true;
 }
 EXPORT_SYMBOL(skb_flow_dissect);
+
+static u32 hashrnd __read_mostly;
+
+/*
+ * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
+ * and src/dst port numbers.  Sets rxhash in skb to non-zero hash value
+ * on success, zero indicates no valid hash.  Also, sets l4_rxhash in skb
+ * if hash is a canonical 4-tuple hash over transport ports.
+ */
+void __skb_get_rxhash(struct sk_buff *skb)
+{
+       struct flow_keys keys;
+       u32 hash;
+
+       if (!skb_flow_dissect(skb, &keys))
+               return;
+
+       if (keys.ports)
+               skb->l4_rxhash = 1;
+
+       /* get a consistent hash (same value on both flow directions) */
+       if (((__force u32)keys.dst < (__force u32)keys.src) ||
+           (((__force u32)keys.dst == (__force u32)keys.src) &&
+            ((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) {
+               swap(keys.dst, keys.src);
+               swap(keys.port16[0], keys.port16[1]);
+       }
+
+       hash = jhash_3words((__force u32)keys.dst,
+                           (__force u32)keys.src,
+                           (__force u32)keys.ports, hashrnd);
+       if (!hash)
+               hash = 1;
+
+       skb->rxhash = hash;
+}
+EXPORT_SYMBOL(__skb_get_rxhash);
+
+/*
+ * Returns a Tx hash based on the given packet descriptor a Tx queues' number
+ * to be used as a distribution range.
+ */
+u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
+                 unsigned int num_tx_queues)
+{
+       u32 hash;
+       u16 qoffset = 0;
+       u16 qcount = num_tx_queues;
+
+       if (skb_rx_queue_recorded(skb)) {
+               hash = skb_get_rx_queue(skb);
+               while (unlikely(hash >= num_tx_queues))
+                       hash -= num_tx_queues;
+               return hash;
+       }
+
+       if (dev->num_tc) {
+               u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
+               qoffset = dev->tc_to_txq[tc].offset;
+               qcount = dev->tc_to_txq[tc].count;
+       }
+
+       if (skb->sk && skb->sk->sk_hash)
+               hash = skb->sk->sk_hash;
+       else
+               hash = (__force u16) skb->protocol;
+       hash = jhash_1word(hash, hashrnd);
+
+       return (u16) (((u64) hash * qcount) >> 32) + qoffset;
+}
+EXPORT_SYMBOL(__skb_tx_hash);
+
+static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
+{
+       if (unlikely(queue_index >= dev->real_num_tx_queues)) {
+               net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
+                                    dev->name, queue_index,
+                                    dev->real_num_tx_queues);
+               return 0;
+       }
+       return queue_index;
+}
+
+static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
+{
+#ifdef CONFIG_XPS
+       struct xps_dev_maps *dev_maps;
+       struct xps_map *map;
+       int queue_index = -1;
+
+       rcu_read_lock();
+       dev_maps = rcu_dereference(dev->xps_maps);
+       if (dev_maps) {
+               map = rcu_dereference(
+                   dev_maps->cpu_map[raw_smp_processor_id()]);
+               if (map) {
+                       if (map->len == 1)
+                               queue_index = map->queues[0];
+                       else {
+                               u32 hash;
+                               if (skb->sk && skb->sk->sk_hash)
+                                       hash = skb->sk->sk_hash;
+                               else
+                                       hash = (__force u16) skb->protocol ^
+                                           skb->rxhash;
+                               hash = jhash_1word(hash, hashrnd);
+                               queue_index = map->queues[
+                                   ((u64)hash * map->len) >> 32];
+                       }
+                       if (unlikely(queue_index >= dev->real_num_tx_queues))
+                               queue_index = -1;
+               }
+       }
+       rcu_read_unlock();
+
+       return queue_index;
+#else
+       return -1;
+#endif
+}
+
+u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
+{
+       struct sock *sk = skb->sk;
+       int queue_index = sk_tx_queue_get(sk);
+
+       if (queue_index < 0 || skb->ooo_okay ||
+           queue_index >= dev->real_num_tx_queues) {
+               int new_index = get_xps_queue(dev, skb);
+               if (new_index < 0)
+                       new_index = skb_tx_hash(dev, skb);
+
+               if (queue_index != new_index && sk) {
+                       struct dst_entry *dst =
+                                   rcu_dereference_check(sk->sk_dst_cache, 1);
+
+                       if (dst && skb_dst(skb) == dst)
+                               sk_tx_queue_set(sk, queue_index);
+
+               }
+
+               queue_index = new_index;
+       }
+
+       return queue_index;
+}
+EXPORT_SYMBOL(__netdev_pick_tx);
+
+struct netdev_queue *netdev_pick_tx(struct net_device *dev,
+                                   struct sk_buff *skb)
+{
+       int queue_index = 0;
+
+       if (dev->real_num_tx_queues != 1) {
+               const struct net_device_ops *ops = dev->netdev_ops;
+               if (ops->ndo_select_queue)
+                       queue_index = ops->ndo_select_queue(dev, skb);
+               else
+                       queue_index = __netdev_pick_tx(dev, skb);
+               queue_index = dev_cap_txqueue(dev, queue_index);
+       }
+
+       skb_set_queue_mapping(skb, queue_index);
+       return netdev_get_tx_queue(dev, queue_index);
+}
+
+static int __init initialize_hashrnd(void)
+{
+       get_random_bytes(&hashrnd, sizeof(hashrnd));
+       return 0;
+}
+
+late_initcall_sync(initialize_hashrnd);