tcp: tsq: move tsq_flags close to sk_wmem_alloc

tsq_flags being in the same cache line than sk_wmem_alloc
makes a lot of sense. Both fields are changed from tcp_wfree()
and more generally by various TSQ related functions.

Prior patch made room in struct sock and added sk_tsq_flags,
this patch deletes tsq_flags from struct tcp_sock.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index d8be083ab0b01e8d2a1fe28c54e2d79205dca36c..fc5848dad7a43216b3f124c4afdaa6b64b23910c 100644 (file)
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -186,7 +186,6 @@ struct tcp_sock {
        u32     tsoffset;       /* timestamp offset */
 
        struct list_head tsq_node; /* anchor in tsq_tasklet.head list */
-       unsigned long   tsq_flags;
 
        /* Data for direct copy to user */
        struct {

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 1149b48700a125b03359b4b65fbb3f1d8494e0c4..1ef3165114ba37c17b3b6cc036543b9951c72b59 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -663,9 +663,9 @@ static void tcp_push(struct sock *sk, int flags, int mss_now,
        if (tcp_should_autocork(sk, skb, size_goal)) {
 
                /* avoid atomic op if TSQ_THROTTLED bit is already set */
-               if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {
+               if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
                        NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
-                       set_bit(TSQ_THROTTLED, &tp->tsq_flags);
+                       set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
                }
                /* It is possible TX completion already happened
                 * before we set TSQ_THROTTLED.

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index b50f05905cedc9be6306f57a8ba89ddf8e22b982..30d81f533ada5708a61155b589e3835d9af4128d 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -443,7 +443,7 @@ void tcp_v4_err(struct sk_buff *icmp_skb, u32 info)
                        if (!sock_owned_by_user(sk)) {
                                tcp_v4_mtu_reduced(sk);
                        } else {
-                               if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &tp->tsq_flags))
+                               if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
                                        sock_hold(sk);
                        }
                        goto out;

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 5f04bee4c86ab060d72163243b858864a5a589bb..b45101f3d2bd2e0f0077305a061add4f7ea0de27 100644 (file)
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -767,14 +767,15 @@ static void tcp_tasklet_func(unsigned long data)
        list_for_each_safe(q, n, &list) {
                tp = list_entry(q, struct tcp_sock, tsq_node);
                list_del(&tp->tsq_node);
-               clear_bit(TSQ_QUEUED, &tp->tsq_flags);
 
                sk = (struct sock *)tp;
+               clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
+
                if (!sk->sk_lock.owned &&
-                   test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) {
+                   test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags)) {
                        bh_lock_sock(sk);
                        if (!sock_owned_by_user(sk)) {
-                               clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
+                               clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
                                tcp_tsq_handler(sk);
                        }
                        bh_unlock_sock(sk);
@@ -797,16 +798,15 @@ static void tcp_tasklet_func(unsigned long data)
  */
 void tcp_release_cb(struct sock *sk)
 {
-       struct tcp_sock *tp = tcp_sk(sk);
        unsigned long flags, nflags;
 
        /* perform an atomic operation only if at least one flag is set */
        do {
-               flags = tp->tsq_flags;
+               flags = sk->sk_tsq_flags;
                if (!(flags & TCP_DEFERRED_ALL))
                        return;
                nflags = flags & ~TCP_DEFERRED_ALL;
-       } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
+       } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
 
        if (flags & TCPF_TSQ_DEFERRED)
                tcp_tsq_handler(sk);
@@ -878,7 +878,7 @@ void tcp_wfree(struct sk_buff *skb)
        if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
                goto out;
 
-       for (oval = READ_ONCE(tp->tsq_flags);; oval = nval) {
+       for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) {
                struct tsq_tasklet *tsq;
                bool empty;
 
@@ -886,7 +886,7 @@ void tcp_wfree(struct sk_buff *skb)
                        goto out;
 
                nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED;
-               nval = cmpxchg(&tp->tsq_flags, oval, nval);
+               nval = cmpxchg(&sk->sk_tsq_flags, oval, nval);
                if (nval != oval)
                        continue;
 
@@ -2100,7 +2100,7 @@ static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
                    skb->prev == sk->sk_write_queue.next)
                        return false;
 
-               set_bit(TSQ_THROTTLED, &tcp_sk(sk)->tsq_flags);
+               set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
                /* It is possible TX completion already happened
                 * before we set TSQ_THROTTLED, so we must
                 * test again the condition.
@@ -2241,8 +2241,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
                    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
                        break;
 
-               if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags))
-                       clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
+               if (test_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
+                       clear_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags);
                if (tcp_small_queue_check(sk, skb, 0))
                        break;
 
@@ -3545,8 +3545,6 @@ void tcp_send_ack(struct sock *sk)
        /* We do not want pure acks influencing TCP Small Queues or fq/pacing
         * too much.
         * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
-        * We also avoid tcp_wfree() overhead (cache line miss accessing
-        * tp->tsq_flags) by using regular sock_wfree()
         */
        skb_set_tcp_pure_ack(buff);
 

diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
index 3ea1cf804748da47516c3499b5c75bedfb962a0e..3705075f42c34b4ce5102bd858d5ed55ed7dd4cc 100644 (file)
--- a/net/ipv4/tcp_timer.c
+++ b/net/ipv4/tcp_timer.c
@@ -310,7 +310,7 @@ static void tcp_delack_timer(unsigned long data)
                inet_csk(sk)->icsk_ack.blocked = 1;
                __NET_INC_STATS(sock_net(sk), LINUX_MIB_DELAYEDACKLOCKED);
                /* deleguate our work to tcp_release_cb() */
-               if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &tcp_sk(sk)->tsq_flags))
+               if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, &sk->sk_tsq_flags))
                        sock_hold(sk);
        }
        bh_unlock_sock(sk);
@@ -592,7 +592,7 @@ static void tcp_write_timer(unsigned long data)
                tcp_write_timer_handler(sk);
        } else {
                /* delegate our work to tcp_release_cb() */
-               if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, &tcp_sk(sk)->tsq_flags))
+               if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, &sk->sk_tsq_flags))
                        sock_hold(sk);
        }
        bh_unlock_sock(sk);

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index a2185a214abcdc16942f3154a7e5d9daeec93e7b..73bc8fc68acdc1ed947bfdb7624b3308a9b70806 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -399,7 +399,7 @@ static void tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                if (!sock_owned_by_user(sk))
                        tcp_v6_mtu_reduced(sk);
                else if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED,
-                                          &tp->tsq_flags))
+                                          &sk->sk_tsq_flags))
                        sock_hold(sk);
                goto out;
        }