net: sched: convert qdisc linked list to hashtable

Convert the per-device linked list into a hashtable. The primary
motivation for this change is that currently, we're not tracking all the
qdiscs in hierarchy (e.g. excluding default qdiscs), as the lookup
performed over the linked list by qdisc_match_from_root() is rather
expensive.

The ultimate goal is to get rid of hidden qdiscs completely, which will
bring much more determinism in user experience.

Reviewed-by: Cong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 076df5360ba50544b0d835b1290038cf85be30e4..96e0b6cd964eb7de70cfa0a01c835ec6b6c6376d 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -52,6 +52,7 @@
 #include <uapi/linux/netdevice.h>
 #include <uapi/linux/if_bonding.h>
 #include <uapi/linux/pkt_cls.h>
+#include <linux/hashtable.h>
 
 struct netpoll_info;
 struct device;
@@ -1800,6 +1801,9 @@ struct net_device {
        unsigned int            num_tx_queues;
        unsigned int            real_num_tx_queues;
        struct Qdisc            *qdisc;
+#ifdef CONFIG_NET_SCHED
+       DECLARE_HASHTABLE       (qdisc_hash, 4);
+#endif
        unsigned long           tx_queue_len;
        spinlock_t              tx_global_lock;
        int                     watchdog_timeo;

diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index 7caa99b482c6352d36a8d383e4af2fded040831c..cd334c9584e9aa7267652b9f6f538834a7cbcbfa 100644 (file)
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -90,8 +90,8 @@ int unregister_qdisc(struct Qdisc_ops *qops);
 void qdisc_get_default(char *id, size_t len);
 int qdisc_set_default(const char *id);
 
-void qdisc_list_add(struct Qdisc *q);
-void qdisc_list_del(struct Qdisc *q);
+void qdisc_hash_add(struct Qdisc *q);
+void qdisc_hash_del(struct Qdisc *q);
 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle);
 struct Qdisc *qdisc_lookup_class(struct net_device *dev, u32 handle);
 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 909aff2db2b36b11d4b92f261a5a90e750cd59bc..0d501779cc68f9426e58da6d039dd64adc937c20 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -61,7 +61,7 @@ struct Qdisc {
        u32                     limit;
        const struct Qdisc_ops  *ops;
        struct qdisc_size_table __rcu *stab;
-       struct list_head        list;
+       struct hlist_node       hash;
        u32                     handle;
        u32                     parent;
        void                    *u32_node;

diff --git a/net/core/dev.c b/net/core/dev.c
index 4ce07dc25573ed3d20f181f5b36327cb0f407fe3..936ea0054f5707d23a27d05252a75098165efe54 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -7629,6 +7629,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
        INIT_LIST_HEAD(&dev->all_adj_list.lower);
        INIT_LIST_HEAD(&dev->ptype_all);
        INIT_LIST_HEAD(&dev->ptype_specific);
+#ifdef CONFIG_NET_SCHED
+       hash_init(dev->qdisc_hash);
+#endif
        dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
        setup(dev);
 

diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 12ebde84552337ec7248e56fe1a64edc0227d7ec..25aada7b095cf31c8566104e27f437c34be9e262 100644 (file)
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -29,6 +29,7 @@
 #include <linux/hrtimer.h>
 #include <linux/lockdep.h>
 #include <linux/slab.h>
+#include <linux/hashtable.h>
 
 #include <net/net_namespace.h>
 #include <net/sock.h>
@@ -263,33 +264,33 @@ static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
            root->handle == handle)
                return root;
 
-       list_for_each_entry_rcu(q, &root->list, list) {
+       hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle) {
                if (q->handle == handle)
                        return q;
        }
        return NULL;
 }
 
-void qdisc_list_add(struct Qdisc *q)
+void qdisc_hash_add(struct Qdisc *q)
 {
        if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
                struct Qdisc *root = qdisc_dev(q)->qdisc;
 
                WARN_ON_ONCE(root == &noop_qdisc);
                ASSERT_RTNL();
-               list_add_tail_rcu(&q->list, &root->list);
+               hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
        }
 }
-EXPORT_SYMBOL(qdisc_list_add);
+EXPORT_SYMBOL(qdisc_hash_add);
 
-void qdisc_list_del(struct Qdisc *q)
+void qdisc_hash_del(struct Qdisc *q)
 {
        if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
                ASSERT_RTNL();
-               list_del_rcu(&q->list);
+               hash_del_rcu(&q->hash);
        }
 }
-EXPORT_SYMBOL(qdisc_list_del);
+EXPORT_SYMBOL(qdisc_hash_del);
 
 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
 {
@@ -998,7 +999,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
                                goto err_out4;
                }
 
-               qdisc_list_add(sch);
+               qdisc_hash_add(sch);
 
                return sch;
        }
@@ -1435,6 +1436,7 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
 {
        int ret = 0, q_idx = *q_idx_p;
        struct Qdisc *q;
+       int b;
 
        if (!root)
                return 0;
@@ -1449,7 +1451,7 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
                        goto done;
                q_idx++;
        }
-       list_for_each_entry(q, &root->list, list) {
+       hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
                if (q_idx < s_q_idx) {
                        q_idx++;
                        continue;
@@ -1765,6 +1767,7 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
                               int *t_p, int s_t)
 {
        struct Qdisc *q;
+       int b;
 
        if (!root)
                return 0;
@@ -1772,7 +1775,7 @@ static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
        if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
                return -1;
 
-       list_for_each_entry(q, &root->list, list) {
+       hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
                if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
                        return -1;
        }

diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index e95b67cd571844798de98dc4a97743ed89e0d37b..18faecc3f13e0f60f1ae7cbcff49115e825db42d 100644 (file)
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -423,7 +423,6 @@ struct Qdisc noop_qdisc = {
        .dequeue        =       noop_dequeue,
        .flags          =       TCQ_F_BUILTIN,
        .ops            =       &noop_qdisc_ops,
-       .list           =       LIST_HEAD_INIT(noop_qdisc.list),
        .q.lock         =       __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
        .dev_queue      =       &noop_netdev_queue,
        .running        =       SEQCNT_ZERO(noop_qdisc.running),
@@ -613,7 +612,6 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
                sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
                sch->padded = (char *) sch - (char *) p;
        }
-       INIT_LIST_HEAD(&sch->list);
        skb_queue_head_init(&sch->q);
 
        spin_lock_init(&sch->busylock);
@@ -700,7 +698,7 @@ void qdisc_destroy(struct Qdisc *qdisc)
                return;
 
 #ifdef CONFIG_NET_SCHED
-       qdisc_list_del(qdisc);
+       qdisc_hash_del(qdisc);
 
        qdisc_put_stab(rtnl_dereference(qdisc->stab));
 #endif
@@ -788,6 +786,10 @@ static void attach_default_qdiscs(struct net_device *dev)
                        qdisc->ops->attach(qdisc);
                }
        }
+#ifdef CONFIG_NET_SCHED
+       if (dev->qdisc)
+               qdisc_hash_add(dev->qdisc);
+#endif
 }
 
 static void transition_one_qdisc(struct net_device *dev,

diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index b9439827c172d1adaf27f413bcc77d030b6784d7..2bc8d7f8df161005bc89245ca5ebc52f3360e3af 100644 (file)
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -88,7 +88,7 @@ static void mq_attach(struct Qdisc *sch)
                        qdisc_destroy(old);
 #ifdef CONFIG_NET_SCHED
                if (ntx < dev->real_num_tx_queues)
-                       qdisc_list_add(qdisc);
+                       qdisc_hash_add(qdisc);
 #endif
 
        }

diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index 549c6635992444d08652b7876262aa9f69e5c99d..b5c502c781439c0440c088b000a7b859271e50b2 100644 (file)
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -182,7 +182,7 @@ static void mqprio_attach(struct Qdisc *sch)
                if (old)
                        qdisc_destroy(old);
                if (ntx < dev->real_num_tx_queues)
-                       qdisc_list_add(qdisc);
+                       qdisc_hash_add(qdisc);
        }
        kfree(priv->qdiscs);
        priv->qdiscs = NULL;