udp: avoid a cache miss on dequeue

Since UDP no more uses sk->destructor, we can clear completely
the skb head state before enqueuing. Amend and use
skb_release_head_state() for that.

All head states share a single cacheline, which is not
normally used/accesses on dequeue. We can avoid entirely accessing
such cacheline implementing and using in the UDP code a specialized
skb free helper which ignores the skb head state.

This saves a cacheline miss at skb deallocation time.

v1 -> v2:
  replaced secpath_reset() with skb_release_head_state()

Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index decce3655a48e0f94d3fc93a0e91dd418de04725..d66d4feaac867633af0162065e105caf8f4cb383 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -880,10 +880,12 @@ static inline bool skb_unref(struct sk_buff *skb)
        return true;
 }
 
+void skb_release_head_state(struct sk_buff *skb);
 void kfree_skb(struct sk_buff *skb);
 void kfree_skb_list(struct sk_buff *segs);
 void skb_tx_error(struct sk_buff *skb);
 void consume_skb(struct sk_buff *skb);
+void consume_stateless_skb(struct sk_buff *skb);
 void  __kfree_skb(struct sk_buff *skb);
 extern struct kmem_cache *skbuff_head_cache;
 

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 747263c498388cfdf49e6891869dadee62242ddf..304602784c3b78249c73609e8aad9cbc55e70e66 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -643,12 +643,10 @@ fastpath:
        kmem_cache_free(skbuff_fclone_cache, fclones);
 }
 
-static void skb_release_head_state(struct sk_buff *skb)
+void skb_release_head_state(struct sk_buff *skb)
 {
        skb_dst_drop(skb);
-#ifdef CONFIG_XFRM
-       secpath_put(skb->sp);
-#endif
+       secpath_reset(skb);
        if (skb->destructor) {
                WARN_ON(in_irq());
                skb->destructor(skb);
@@ -751,6 +749,24 @@ void consume_skb(struct sk_buff *skb)
 }
 EXPORT_SYMBOL(consume_skb);
 
+/**
+ *     consume_stateless_skb - free an skbuff, assuming it is stateless
+ *     @skb: buffer to free
+ *
+ *     Works like consume_skb(), but this variant assumes that all the head
+ *     states have been already dropped.
+ */
+void consume_stateless_skb(struct sk_buff *skb)
+{
+       if (!skb_unref(skb))
+               return;
+
+       trace_consume_skb(skb);
+       if (likely(skb->head))
+               skb_release_data(skb);
+       kfree_skbmem(skb);
+}
+
 void __kfree_skb_flush(void)
 {
        struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index fdcb7437cc1541d87dcf5bb2c861077473901499..d8b265f1a33be9aed413dedcd737193457bf71ee 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1359,7 +1359,8 @@ void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
                sk_peek_offset_bwd(sk, len);
                unlock_sock_fast(sk, slow);
        }
-       consume_skb(skb);
+
+       consume_stateless_skb(skb);
 }
 EXPORT_SYMBOL_GPL(skb_consume_udp);
 
@@ -1739,6 +1740,9 @@ static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
                sk_mark_napi_id_once(sk, skb);
        }
 
+       /* clear all pending head states while they are hot in the cache */
+       skb_release_head_state(skb);
+
        rc = __udp_enqueue_schedule_skb(sk, skb);
        if (rc < 0) {
                int is_udplite = IS_UDPLITE(sk);