netvm: propagate page->pfmemalloc from skb_alloc_page to skb

The skb->pfmemalloc flag gets set to true iff during the slab allocation
of data in __alloc_skb that the the PFMEMALLOC reserves were used.  If
page splitting is used, it is possible that pages will be allocated from
the PFMEMALLOC reserve without propagating this information to the skb.
This patch propagates page->pfmemalloc from pages allocated for fragments
to the skb.

It works by reintroducing and expanding the skb_alloc_page() API to take
an skb.  If the page was allocated from pfmemalloc reserves, it is
automatically copied.  If the driver allocates the page before the skb, it
should call skb_propagate_pfmemalloc() after the skb is allocated to
ensure the flag is copied properly.

Failure to do so is not critical.  The resulting driver may perform slower
if it is used for swap-over-NBD or swap-over-NFS but it should not result
in failure.

[davem@davemloft.net: API rename and consistency]
Signed-off-by: Mel Gorman <mgorman@suse.de>
Acked-by: David S. Miller <davem@davemloft.net>
Cc: Neil Brown <neilb@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Christie <michaelc@cs.wisc.edu>
Cc: Eric B Munson <emunson@mgebm.net>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Christoph Lameter <cl@linux.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index 8596acaa402b4af43d3d0b9c59ac2dd107f478c1..d49933ed551f7a4fca4d6b71f0528aaf9f6a80d6 100644 (file)
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -528,7 +528,7 @@ static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n,
 #endif
 
        while (n--) {
-               pg = alloc_page(gfp);
+               pg = __skb_alloc_page(gfp, NULL);
                if (unlikely(!pg)) {
                        q->alloc_failed++;
                        break;

diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
index f2d1ecdcaf98efe0801de112f36ba12ef81fbefd..8877fbfefb639ed5e7d6282981dd6f722dd6f5f5 100644 (file)
--- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
@@ -653,7 +653,7 @@ static unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl,
 
 alloc_small_pages:
        while (n--) {
-               page = alloc_page(gfp | __GFP_NOWARN | __GFP_COLD);
+               page = __skb_alloc_page(gfp | __GFP_NOWARN, NULL);
                if (unlikely(!page)) {
                        fl->alloc_failed++;
                        break;

diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 1050411e7ca337f8cd28dff367d5cb2e94838ddf..b7c2d5050572828c8d07e228b2e44235e9f01801 100644 (file)
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -6235,7 +6235,7 @@ static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
                return true;
 
        if (!page) {
-               page = alloc_page(GFP_ATOMIC | __GFP_COLD);
+               page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
                bi->page = page;
                if (unlikely(!page)) {
                        rx_ring->rx_stats.alloc_failed++;

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index c709eae58c630a9a1e2c5e5e7539d733cf609e31..4326f74f7137a333461683f00c8fc1cadfcee4b4 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -1141,8 +1141,8 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
 
        /* alloc new page for storage */
        if (likely(!page)) {
-               page = alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
-                                  ixgbe_rx_pg_order(rx_ring));
+               page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
+                                        bi->skb, ixgbe_rx_pg_order(rx_ring));
                if (unlikely(!page)) {
                        rx_ring->rx_stats.alloc_rx_page_failed++;
                        return false;

diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 3f9841d619adae00002d3f662f5067f3b6092d63..60ef64587412a6a70af056f93e253f4fe407cce2 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -352,7 +352,6 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
                                adapter->alloc_rx_buff_failed++;
                                goto no_buffers;
                        }
-
                        bi->skb = skb;
                }
                if (!bi->dma) {

diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c
index 187c144c5e5bc23f3fd9b6286374349f7883862f..64610048ce87c178770059b163395a285cb69e77 100644 (file)
--- a/drivers/net/usb/cdc-phonet.c
+++ b/drivers/net/usb/cdc-phonet.c
@@ -130,7 +130,7 @@ static int rx_submit(struct usbpn_dev *pnd, struct urb *req, gfp_t gfp_flags)
        struct page *page;
        int err;
 
-       page = alloc_page(gfp_flags);
+       page = __skb_alloc_page(gfp_flags | __GFP_NOMEMALLOC, NULL);
        if (!page)
                return -ENOMEM;
 

diff --git a/drivers/usb/gadget/f_phonet.c b/drivers/usb/gadget/f_phonet.c
index 965a6293206acc5bb52facb12d8acf50d3c3c6a2..8ee9268fe253f2dc01de4f84f451ef3a1870a69a 100644 (file)
--- a/drivers/usb/gadget/f_phonet.c
+++ b/drivers/usb/gadget/f_phonet.c
@@ -301,7 +301,7 @@ pn_rx_submit(struct f_phonet *fp, struct usb_request *req, gfp_t gfp_flags)
        struct page *page;
        int err;
 
-       page = alloc_page(gfp_flags);
+       page = __skb_alloc_page(gfp_flags | __GFP_NOMEMALLOC, NULL);
        if (!page)
                return -ENOMEM;
 

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index b814bb8fd5aba0afe78ff98227aff373519a9a9f..7632c87da2c9fd66eab61b6c7415d5669b7aeefd 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1774,6 +1774,61 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
        return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
 }
 
+/*
+ *     __skb_alloc_page - allocate pages for ps-rx on a skb and preserve pfmemalloc data
+ *     @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX
+ *     @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used
+ *     @order: size of the allocation
+ *
+ *     Allocate a new page.
+ *
+ *     %NULL is returned if there is no free memory.
+*/
+static inline struct page *__skb_alloc_pages(gfp_t gfp_mask,
+                                             struct sk_buff *skb,
+                                             unsigned int order)
+{
+       struct page *page;
+
+       gfp_mask |= __GFP_COLD;
+
+       if (!(gfp_mask & __GFP_NOMEMALLOC))
+               gfp_mask |= __GFP_MEMALLOC;
+
+       page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
+       if (skb && page && page->pfmemalloc)
+               skb->pfmemalloc = true;
+
+       return page;
+}
+
+/**
+ *     __skb_alloc_page - allocate a page for ps-rx for a given skb and preserve pfmemalloc data
+ *     @gfp_mask: alloc_pages_node mask. Set __GFP_NOMEMALLOC if not for network packet RX
+ *     @skb: skb to set pfmemalloc on if __GFP_MEMALLOC is used
+ *
+ *     Allocate a new page.
+ *
+ *     %NULL is returned if there is no free memory.
+ */
+static inline struct page *__skb_alloc_page(gfp_t gfp_mask,
+                                            struct sk_buff *skb)
+{
+       return __skb_alloc_pages(gfp_mask, skb, 0);
+}
+
+/**
+ *     skb_propagate_pfmemalloc - Propagate pfmemalloc if skb is allocated after RX page
+ *     @page: The page that was allocated from skb_alloc_page
+ *     @skb: The skb that may need pfmemalloc set
+ */
+static inline void skb_propagate_pfmemalloc(struct page *page,
+                                            struct sk_buff *skb)
+{
+       if (page && page->pfmemalloc)
+               skb->pfmemalloc = true;
+}
+
 /**
  * skb_frag_page - retrieve the page refered to by a paged fragment
  * @frag: the paged fragment