[PATCH] splice: fix page stealing LRU handling.

Originally from Nick Piggin, just adapted to the newer branch.

You can't check PageLRU without holding zone->lru_lock.  The page
release code can get away with it only because the page refcount is 0 at
that point. Also, you can't reliably remove pages from the LRU unless
the refcount is 0. Ever.

Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Jens Axboe <axboe@suse.de>

diff --git a/fs/pipe.c b/fs/pipe.c
index 5093408746b8f0d16234b11e9e2c1620118638ee..795df987cd38bbbe773fcc029084c019c75e5d02 100644 (file)
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -95,6 +95,8 @@ static void anon_pipe_buf_release(struct pipe_inode_info *info, struct pipe_buff
 {
        struct page *page = buf->page;
 
+       buf->flags &= ~PIPE_BUF_FLAG_STOLEN;
+
        /*
         * If nobody else uses this page, and we don't already have a
         * temporary page, let's keep track of it as a one-deep
@@ -124,6 +126,7 @@ static void anon_pipe_buf_unmap(struct pipe_inode_info *info, struct pipe_buffer
 static int anon_pipe_buf_steal(struct pipe_inode_info *info,
                               struct pipe_buffer *buf)
 {
+       buf->flags |= PIPE_BUF_FLAG_STOLEN;
        return 0;
 }
 

diff --git a/fs/splice.c b/fs/splice.c
index b5fb2f3e3ac6c3525c6567f56288ea4553e2568d..bfa42a277bb874ac94a34db4d3c7f593afc5410c 100644 (file)
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -67,16 +67,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
        if (!remove_mapping(mapping, page))
                return 1;
 
-       if (PageLRU(page)) {
-               struct zone *zone = page_zone(page);
-
-               spin_lock_irq(&zone->lru_lock);
-               BUG_ON(!PageLRU(page));
-               __ClearPageLRU(page);
-               del_page_from_lru(zone, page);
-               spin_unlock_irq(&zone->lru_lock);
-       }
-
+       buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU;
        return 0;
 }
 
@@ -85,6 +76,7 @@ static void page_cache_pipe_buf_release(struct pipe_inode_info *info,
 {
        page_cache_release(buf->page);
        buf->page = NULL;
+       buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
 }
 
 static void *page_cache_pipe_buf_map(struct file *file,
@@ -414,11 +406,12 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
 {
        struct file *file = sd->file;
        struct address_space *mapping = file->f_mapping;
+       gfp_t gfp_mask = mapping_gfp_mask(mapping);
        unsigned int offset;
        struct page *page;
        pgoff_t index;
        char *src;
-       int ret, stolen;
+       int ret;
 
        /*
         * after this, page will be locked and unmapped
@@ -429,7 +422,6 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
 
        index = sd->pos >> PAGE_CACHE_SHIFT;
        offset = sd->pos & ~PAGE_CACHE_MASK;
-       stolen = 0;
 
        /*
         * reuse buf page, if SPLICE_F_MOVE is set
@@ -443,15 +435,15 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf,
                        goto find_page;
 
                page = buf->page;
-               stolen = 1;
-               if (add_to_page_cache_lru(page, mapping, index,
-                                               mapping_gfp_mask(mapping)))
+               if (add_to_page_cache(page, mapping, index, gfp_mask))
                        goto find_page;
+
+               if (!(buf->flags & PIPE_BUF_FLAG_LRU))
+                       lru_cache_add(page);
        } else {
 find_page:
                ret = -ENOMEM;
-               page = find_or_create_page(mapping, index,
-                                               mapping_gfp_mask(mapping));
+               page = find_or_create_page(mapping, index, gfp_mask);
                if (!page)
                        goto out;
 
@@ -494,7 +486,7 @@ find_page:
        } else if (ret)
                goto out;
 
-       if (!stolen) {
+       if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
                char *dst = kmap_atomic(page, KM_USER0);
 
                memcpy(dst + offset, src + buf->offset, sd->len);
@@ -511,7 +503,7 @@ find_page:
 
        balance_dirty_pages_ratelimited(mapping);
 out:
-       if (!stolen) {
+       if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
                page_cache_release(page);
                unlock_page(page);
        }

diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h
index 70ae9332ec16ee64709908f8cc854532cb612616..ec384958d509690090aa1936b4624cb3da05b2d8 100644 (file)
--- a/include/linux/pipe_fs_i.h
+++ b/include/linux/pipe_fs_i.h
@@ -5,10 +5,14 @@
 
 #define PIPE_BUFFERS (16)
 
+#define PIPE_BUF_FLAG_STOLEN   0x01
+#define PIPE_BUF_FLAG_LRU      0x02
+
 struct pipe_buffer {
        struct page *page;
        unsigned int offset, len;
        struct pipe_buf_operations *ops;
+       unsigned int flags;
 };
 
 struct pipe_buf_operations {