net: reorder struct Qdisc for better SMP performance

dev_queue_xmit() needs to dirty fields "state", "q", "bstats" and "qstats"

On x86_64 arch, they currently span three cache lines, involving more
cache line ping pongs than necessary, making longer holding of queue spinlock.

We can reduce this to one cache line, by grouping all read-mostly fields
at the beginning of structure. (Or should I say, all highly modified fields
at the end :) )

Before patch :

offsetof(struct Qdisc, state)=0x38
offsetof(struct Qdisc, q)=0x48
offsetof(struct Qdisc, bstats)=0x80
offsetof(struct Qdisc, qstats)=0x90
sizeof(struct Qdisc)=0xc8

After patch :

offsetof(struct Qdisc, state)=0x80
offsetof(struct Qdisc, q)=0x88
offsetof(struct Qdisc, bstats)=0xa0
offsetof(struct Qdisc, qstats)=0xac
sizeof(struct Qdisc)=0xc0

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/gen_stats.h b/include/linux/gen_stats.h
index 13f4e74609ac10747eceffb84f3f1293a83e71a7..0ffa41df0ee8ca007d9cda5b40fdbe315ee0a845 100644 (file)
--- a/include/linux/gen_stats.h
+++ b/include/linux/gen_stats.h
@@ -22,7 +22,7 @@ struct gnet_stats_basic
 {
        __u64   bytes;
        __u32   packets;
-};
+} __attribute__ ((packed));
 
 /**
  * struct gnet_stats_rate_est - rate estimator

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 3d78a4d224601f5003aef260f4363bc88707d3be..964ffa0d8815139ae22d351590b5673a4dd38850 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -49,18 +49,10 @@ struct Qdisc
        int                     padded;
        struct Qdisc_ops        *ops;
        struct qdisc_size_table *stab;
+       struct list_head        list;
        u32                     handle;
        u32                     parent;
        atomic_t                refcnt;
-       unsigned long           state;
-       struct sk_buff          *gso_skb;
-       struct sk_buff_head     q;
-       struct netdev_queue     *dev_queue;
-       struct Qdisc            *next_sched;
-       struct list_head        list;
-
-       struct gnet_stats_basic bstats;
-       struct gnet_stats_queue qstats;
        struct gnet_stats_rate_est      rate_est;
        int                     (*reshape_fail)(struct sk_buff *skb,
                                        struct Qdisc *q);
@@ -71,6 +63,17 @@ struct Qdisc
         * and it will live until better solution will be invented.
         */
        struct Qdisc            *__parent;
+       struct netdev_queue     *dev_queue;
+       struct Qdisc            *next_sched;
+
+       struct sk_buff          *gso_skb;
+       /*
+        * For performance sake on SMP, we put highly modified fields at the end
+        */
+       unsigned long           state;
+       struct sk_buff_head     q;
+       struct gnet_stats_basic bstats;
+       struct gnet_stats_queue qstats;
 };
 
 struct Qdisc_class_ops