fib: avoid false sharing on fib_table_hash

While doing profile analysis, I found fib_hash_table was sometime in a
cache line shared by a possibly often written kernel structure.

(CONFIG_IP_ROUTE_MULTIPATH || !CONFIG_IPV6_MULTIPLE_TABLES)

It's hard to detect because not easily reproductible.

Make sure we allocate a full cache line to keep this shared in all cpus
caches.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c
index 919f2ad19b4973eecfec44b39a9c6376e751ecae..3df057e896403a9e557ac0dd65688874fe4ac0df 100644 (file)
--- a/net/ipv4/fib_frontend.c
+++ b/net/ipv4/fib_frontend.c
@@ -1016,16 +1016,15 @@ static struct notifier_block fib_netdev_notifier = {
 static int __net_init ip_fib_net_init(struct net *net)
 {
        int err;
-       unsigned int i;
+       size_t size = sizeof(struct hlist_head) * FIB_TABLE_HASHSZ;
+
+       /* Avoid false sharing : Use at least a full cache line */
+       size = max_t(size_t, size, L1_CACHE_BYTES);
 
-       net->ipv4.fib_table_hash = kzalloc(
-                       sizeof(struct hlist_head)*FIB_TABLE_HASHSZ, GFP_KERNEL);
+       net->ipv4.fib_table_hash = kzalloc(size, GFP_KERNEL);
        if (net->ipv4.fib_table_hash == NULL)
                return -ENOMEM;
 
-       for (i = 0; i < FIB_TABLE_HASHSZ; i++)
-               INIT_HLIST_HEAD(&net->ipv4.fib_table_hash[i]);
-
        err = fib4_rules_init(net);
        if (err < 0)
                goto fail;

diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c
index b6a585909d3560d0f9c0396fe8ed4b136c0ef5f6..de382114609b7de8d0cbf5c5447a18d4f5a89a2d 100644 (file)
--- a/net/ipv6/ip6_fib.c
+++ b/net/ipv6/ip6_fib.c
@@ -1500,15 +1500,18 @@ static void fib6_gc_timer_cb(unsigned long arg)
 
 static int __net_init fib6_net_init(struct net *net)
 {
+       size_t size = sizeof(struct hlist_head) * FIB6_TABLE_HASHSZ;
+
        setup_timer(&net->ipv6.ip6_fib_timer, fib6_gc_timer_cb, (unsigned long)net);
 
        net->ipv6.rt6_stats = kzalloc(sizeof(*net->ipv6.rt6_stats), GFP_KERNEL);
        if (!net->ipv6.rt6_stats)
                goto out_timer;
 
-       net->ipv6.fib_table_hash = kcalloc(FIB6_TABLE_HASHSZ,
-                                          sizeof(*net->ipv6.fib_table_hash),
-                                          GFP_KERNEL);
+       /* Avoid false sharing : Use at least a full cache line */
+       size = max_t(size_t, size, L1_CACHE_BYTES);
+
+       net->ipv6.fib_table_hash = kzalloc(size, GFP_KERNEL);
        if (!net->ipv6.fib_table_hash)
                goto out_rt6_stats;