udp: Use hash2 for long hash1 chains in __udp*_lib_mcast_deliver.

Many multicast sources can have the same port which can result in a very
large list when hashing by port only. Hash by address and port instead
if this is the case. This makes multicast more similar to unicast.

On a 24-core machine receiving from 500 multicast sockets on the same
port, before this patch 80% of system CPU was used up by spin locking
and only ~25% of packets were successfully delivered.

With this patch, all packets are delivered and kernel overhead is ~8%
system CPU on spinlocks.

Signed-off-by: David Held <drheld@google.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/sock.h b/include/net/sock.h
index 29e48a6d1dedd20ef9756ea61b1dba0cfb47e4a4..28f734601b506e57e5985f47005e0ee2e22b42f9 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -660,6 +660,20 @@ static inline void sk_add_bind_node(struct sock *sk,
 #define sk_for_each_bound(__sk, list) \
        hlist_for_each_entry(__sk, list, sk_bind_node)
 
+/**
+ * sk_nulls_for_each_entry_offset - iterate over a list at a given struct offset
+ * @tpos:      the type * to use as a loop cursor.
+ * @pos:       the &struct hlist_node to use as a loop cursor.
+ * @head:      the head for your list.
+ * @offset:    offset of hlist_node within the struct.
+ *
+ */
+#define sk_nulls_for_each_entry_offset(tpos, pos, head, offset)                       \
+       for (pos = (head)->first;                                              \
+            (!is_a_nulls(pos)) &&                                             \
+               ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;});       \
+            pos = pos->next)
+
 static inline struct user_namespace *sk_user_ns(struct sock *sk)
 {
        /* Careful only use this in a context where these parameters

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index bbcc33737ef17cd473faa1d99713a0b367e3dfc3..f31053b90ee069d38942ef5d69f459d5ac6da0b1 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1619,6 +1619,8 @@ static void flush_stack(struct sock **stack, unsigned int count,
 
                if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)
                        skb1 = NULL;
+
+               sock_put(sk);
        }
        if (unlikely(skb1))
                kfree_skb(skb1);
@@ -1651,10 +1653,20 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
        unsigned short hnum = ntohs(uh->dest);
        struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
        int dif = skb->dev->ifindex;
-       unsigned int i, count = 0;
+       unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
+       unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
+
+       if (use_hash2) {
+               hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
+                           udp_table.mask;
+               hash2 = udp4_portaddr_hash(net, daddr, hnum) & udp_table.mask;
+start_lookup:
+               hslot = &udp_table.hash2[hash2];
+               offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
+       }
 
        spin_lock(&hslot->lock);
-       sk_nulls_for_each(sk, node, &hslot->head) {
+       sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {
                if (__udp_is_mcast_sock(net, sk,
                                        uh->dest, daddr,
                                        uh->source, saddr,
@@ -1664,24 +1676,23 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
                                count = 0;
                        }
                        stack[count++] = sk;
+                       sock_hold(sk);
                }
        }
-       /*
-        * before releasing chain lock, we must take a reference on sockets
-        */
-       for (i = 0; i < count; i++)
-               sock_hold(stack[i]);
 
        spin_unlock(&hslot->lock);
 
+       /* Also lookup *:port if we are using hash2 and haven't done so yet. */
+       if (use_hash2 && hash2 != hash2_any) {
+               hash2 = hash2_any;
+               goto start_lookup;
+       }
+
        /*
         * do the slow work with no lock held
         */
        if (count) {
                flush_stack(stack, count, skb, count - 1);
-
-               for (i = 0; i < count; i++)
-                       sock_put(stack[i]);
        } else {
                kfree_skb(skb);
        }

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 7d3bd80085be10637bd862b5c241aed25d297e5d..f9d8800bb72fc43d7a83ec30abb7d7d41fd17978 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -745,6 +745,7 @@ static void flush_stack(struct sock **stack, unsigned int count,
 
                if (skb1 && udpv6_queue_rcv_skb(sk, skb1) <= 0)
                        skb1 = NULL;
+               sock_put(sk);
        }
        if (unlikely(skb1))
                kfree_skb(skb1);
@@ -774,10 +775,20 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
        unsigned short hnum = ntohs(uh->dest);
        struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
        int dif = inet6_iif(skb);
-       unsigned int i, count = 0;
+       unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
+       unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
+
+       if (use_hash2) {
+               hash2_any = udp6_portaddr_hash(net, &in6addr_any, hnum) &
+                           udp_table.mask;
+               hash2 = udp6_portaddr_hash(net, daddr, hnum) & udp_table.mask;
+start_lookup:
+               hslot = &udp_table.hash2[hash2];
+               offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
+       }
 
        spin_lock(&hslot->lock);
-       sk_nulls_for_each(sk, node, &hslot->head) {
+       sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {
                if (__udp_v6_is_mcast_sock(net, sk,
                                           uh->dest, daddr,
                                           uh->source, saddr,
@@ -791,21 +802,20 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
                                count = 0;
                        }
                        stack[count++] = sk;
+                       sock_hold(sk);
                }
        }
-       /*
-        * before releasing the lock, we must take reference on sockets
-        */
-       for (i = 0; i < count; i++)
-               sock_hold(stack[i]);
 
        spin_unlock(&hslot->lock);
 
+       /* Also lookup *:port if we are using hash2 and haven't done so yet. */
+       if (use_hash2 && hash2 != hash2_any) {
+               hash2 = hash2_any;
+               goto start_lookup;
+       }
+
        if (count) {
                flush_stack(stack, count, skb, count - 1);
-
-               for (i = 0; i < count; i++)
-                       sock_put(stack[i]);
        } else {
                kfree_skb(skb);
        }