net: introduce __skb_header_release()

While profiling TCP stack, I noticed one useless atomic operation
in tcp_sendmsg(), caused by skb_header_release().

It turns out all current skb_header_release() users have a fresh skb,
that no other user can see, so we can avoid one atomic operation.

Introduce __skb_header_release() to clearly document this.

This gave me a 1.5 % improvement on TCP_RR workload.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index f1bfa3781c758ee7287089be07c7850876ec8b8a..8eaa62400fca0bc5b7ed9dee0d83efcc5e366e21 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1083,6 +1083,7 @@ static inline int skb_header_cloned(const struct sk_buff *skb)
  *     Drop a reference to the header part of the buffer.  This is done
  *     by acquiring a payload reference.  You must not read from the header
  *     part of skb->data after this.
+ *     Note : Check if you can use __skb_header_release() instead.
  */
 static inline void skb_header_release(struct sk_buff *skb)
 {
@@ -1091,6 +1092,20 @@ static inline void skb_header_release(struct sk_buff *skb)
        atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
 }
 
+/**
+ *     __skb_header_release - release reference to header
+ *     @skb: buffer to operate on
+ *
+ *     Variant of skb_header_release() assuming skb is private to caller.
+ *     We can avoid one atomic operation.
+ */
+static inline void __skb_header_release(struct sk_buff *skb)
+{
+       skb->nohdr = 1;
+       atomic_set(&skb_shinfo(skb)->dataref, 1 + (1 << SKB_DATAREF_SHIFT));
+}
+
+
 /**
  *     skb_shared - is the buffer shared
  *     @skb: buffer to check

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 06a8feb10099064d255e8298bb82cf491a2cd025..512dc7dcbc32b9353ef62030c121f81f30a2a1d6 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3179,7 +3179,7 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
        skb_shinfo(nskb)->frag_list = p;
        skb_shinfo(nskb)->gso_size = pinfo->gso_size;
        pinfo->gso_size = 0;
-       skb_header_release(p);
+       __skb_header_release(p);
        NAPI_GRO_CB(nskb)->last = p;
 
        nskb->data_len += p->len;
@@ -3211,7 +3211,7 @@ merge:
        else
                NAPI_GRO_CB(p)->last->next = skb;
        NAPI_GRO_CB(p)->last = skb;
-       skb_header_release(skb);
+       __skb_header_release(skb);
        lp = p;
 
 done:

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 070aeff1b13166d78df3bf84fc20438afd8a6e04..553b01f52f7158ecd5335814e2b1ae5f67f60210 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -609,7 +609,7 @@ static inline bool forced_push(const struct tcp_sock *tp)
        return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 }
 
-static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
+static void skb_entail(struct sock *sk, struct sk_buff *skb)
 {
        struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
@@ -618,7 +618,7 @@ static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
        tcb->seq     = tcb->end_seq = tp->write_seq;
        tcb->tcp_flags = TCPHDR_ACK;
        tcb->sacked  = 0;
-       skb_header_release(skb);
+       __skb_header_release(skb);
        tcp_add_write_queue_tail(sk, skb);
        sk->sk_wmem_queued += skb->truesize;
        sk_mem_charge(sk, skb->truesize);

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 8c61a7c0c88961ff07db6b12949b5befb09f7318..f173b1c4f81508a37ff0085dabb0f0559f76bbe0 100644 (file)
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -995,7 +995,7 @@ static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
 
        /* Advance write_seq and place onto the write_queue. */
        tp->write_seq = TCP_SKB_CB(skb)->end_seq;
-       skb_header_release(skb);
+       __skb_header_release(skb);
        tcp_add_write_queue_tail(sk, skb);
        sk->sk_wmem_queued += skb->truesize;
        sk_mem_charge(sk, skb->truesize);
@@ -1167,7 +1167,7 @@ int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
        }
 
        /* Link BUFF into the send queue. */
-       skb_header_release(buff);
+       __skb_header_release(buff);
        tcp_insert_write_queue_after(skb, buff, sk);
 
        return 0;
@@ -1671,7 +1671,7 @@ static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
        tcp_set_skb_tso_segs(sk, buff, mss_now);
 
        /* Link BUFF into the send queue. */
-       skb_header_release(buff);
+       __skb_header_release(buff);
        tcp_insert_write_queue_after(skb, buff, sk);
 
        return 0;
@@ -2772,7 +2772,7 @@ int tcp_send_synack(struct sock *sk)
                        if (nskb == NULL)
                                return -ENOMEM;
                        tcp_unlink_write_queue(skb, sk);
-                       skb_header_release(nskb);
+                       __skb_header_release(nskb);
                        __tcp_add_write_queue_head(sk, nskb);
                        sk_wmem_free_skb(sk, skb);
                        sk->sk_wmem_queued += nskb->truesize;
@@ -2947,7 +2947,7 @@ static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
        struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 
        tcb->end_seq += skb->len;
-       skb_header_release(skb);
+       __skb_header_release(skb);
        __tcp_add_write_queue_tail(sk, skb);
        sk->sk_wmem_queued += skb->truesize;
        sk_mem_charge(sk, skb->truesize);