net: frag, move LRU list maintenance outside of rwlock

Updating the fragmentation queues LRU (Least-Recently-Used) list,
required taking the hash writer lock.  However, the LRU list isn't
tied to the hash at all, so we can use a separate lock for it.

Original-idea-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h
index e0eec7450f1570440f515ec62d9a75a916868687..3f237db0a426ce1606587da6060ce1da5f306b60 100644 (file)
--- a/include/net/inet_frag.h
+++ b/include/net/inet_frag.h
@@ -6,6 +6,7 @@
 struct netns_frags {
        int                     nqueues;
        struct list_head        lru_list;
+       spinlock_t              lru_lock;
 
        /* The percpu_counter "mem" need to be cacheline aligned.
         *  mem.count must not share cacheline with other writers
@@ -116,4 +117,25 @@ static inline int sum_frag_mem_limit(struct netns_frags *nf)
        return percpu_counter_sum_positive(&nf->mem);
 }
 
+static inline void inet_frag_lru_move(struct inet_frag_queue *q)
+{
+       spin_lock(&q->net->lru_lock);
+       list_move_tail(&q->lru_list, &q->net->lru_list);
+       spin_unlock(&q->net->lru_lock);
+}
+
+static inline void inet_frag_lru_del(struct inet_frag_queue *q)
+{
+       spin_lock(&q->net->lru_lock);
+       list_del(&q->lru_list);
+       spin_unlock(&q->net->lru_lock);
+}
+
+static inline void inet_frag_lru_add(struct netns_frags *nf,
+                                    struct inet_frag_queue *q)
+{
+       spin_lock(&nf->lru_lock);
+       list_add_tail(&q->lru_list, &nf->lru_list);
+       spin_unlock(&nf->lru_lock);
+}
 #endif

diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c
index b825205650984418dcc56b4a55fdf6e8ac9dccfc..2e453bde69922dfd204d99d0fe2f1cf561149269 100644 (file)
--- a/net/ipv4/inet_fragment.c
+++ b/net/ipv4/inet_fragment.c
@@ -75,6 +75,7 @@ void inet_frags_init_net(struct netns_frags *nf)
        nf->nqueues = 0;
        init_frag_mem_limit(nf);
        INIT_LIST_HEAD(&nf->lru_list);
+       spin_lock_init(&nf->lru_lock);
 }
 EXPORT_SYMBOL(inet_frags_init_net);
 
@@ -100,9 +101,9 @@ static inline void fq_unlink(struct inet_frag_queue *fq, struct inet_frags *f)
 {
        write_lock(&f->lock);
        hlist_del(&fq->list);
-       list_del(&fq->lru_list);
        fq->net->nqueues--;
        write_unlock(&f->lock);
+       inet_frag_lru_del(fq);
 }
 
 void inet_frag_kill(struct inet_frag_queue *fq, struct inet_frags *f)
@@ -170,16 +171,17 @@ int inet_frag_evictor(struct netns_frags *nf, struct inet_frags *f, bool force)
 
        work = frag_mem_limit(nf) - nf->low_thresh;
        while (work > 0) {
-               read_lock(&f->lock);
+               spin_lock(&nf->lru_lock);
+
                if (list_empty(&nf->lru_list)) {
-                       read_unlock(&f->lock);
+                       spin_unlock(&nf->lru_lock);
                        break;
                }
 
                q = list_first_entry(&nf->lru_list,
                                struct inet_frag_queue, lru_list);
                atomic_inc(&q->refcnt);
-               read_unlock(&f->lock);
+               spin_unlock(&nf->lru_lock);
 
                spin_lock(&q->lock);
                if (!(q->last_in & INET_FRAG_COMPLETE))
@@ -233,9 +235,9 @@ static struct inet_frag_queue *inet_frag_intern(struct netns_frags *nf,
 
        atomic_inc(&qp->refcnt);
        hlist_add_head(&qp->list, &f->hash[hash]);
-       list_add_tail(&qp->lru_list, &nf->lru_list);
        nf->nqueues++;
        write_unlock(&f->lock);
+       inet_frag_lru_add(nf, qp);
        return qp;
 }
 

diff --git a/net/ipv4/ip_fragment.c b/net/ipv4/ip_fragment.c
index 927fe58caf46ff78841e17d9c0cd924023675faa..1211613c6c3401f520e8eb2209435e4b71360684 100644 (file)
--- a/net/ipv4/ip_fragment.c
+++ b/net/ipv4/ip_fragment.c
@@ -529,9 +529,7 @@ found:
            qp->q.meat == qp->q.len)
                return ip_frag_reasm(qp, prev, dev);
 
-       write_lock(&ip4_frags.lock);
-       list_move_tail(&qp->q.lru_list, &qp->q.net->lru_list);
-       write_unlock(&ip4_frags.lock);
+       inet_frag_lru_move(&qp->q);
        return -EINPROGRESS;
 
 err:

diff --git a/net/ipv6/netfilter/nf_conntrack_reasm.c b/net/ipv6/netfilter/nf_conntrack_reasm.c
index 07ef2949c9463b95218ab6aac0be6f9a34f35400..c674f158efa8bedfcda37e4e66c5f8e7deb79202 100644 (file)
--- a/net/ipv6/netfilter/nf_conntrack_reasm.c
+++ b/net/ipv6/netfilter/nf_conntrack_reasm.c
@@ -328,9 +328,8 @@ found:
                fq->nhoffset = nhoff;
                fq->q.last_in |= INET_FRAG_FIRST_IN;
        }
-       write_lock(&nf_frags.lock);
-       list_move_tail(&fq->q.lru_list, &fq->q.net->lru_list);
-       write_unlock(&nf_frags.lock);
+
+       inet_frag_lru_move(&fq->q);
        return 0;
 
 discard_fq:

diff --git a/net/ipv6/reassembly.c b/net/ipv6/reassembly.c
index 18cb8defcdb29bfb39d43fc8cfec372d69db9ac1..bab2c270f29207c37ed7b107b6d25c6e12a6053d 100644 (file)
--- a/net/ipv6/reassembly.c
+++ b/net/ipv6/reassembly.c
@@ -341,9 +341,7 @@ found:
            fq->q.meat == fq->q.len)
                return ip6_frag_reasm(fq, prev, dev);
 
-       write_lock(&ip6_frags.lock);
-       list_move_tail(&fq->q.lru_list, &fq->q.net->lru_list);
-       write_unlock(&ip6_frags.lock);
+       inet_frag_lru_move(&fq->q);
        return -1;
 
 discard_fq: