2 * page.c - buffer/page management specific to NILFS
4 * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 * Written by Ryusuke Konishi <ryusuke@osrg.net>,
21 * Seiji Kihara <kihara@osrg.net>.
24 #include <linux/pagemap.h>
25 #include <linux/writeback.h>
26 #include <linux/swap.h>
27 #include <linux/bitops.h>
28 #include <linux/page-flags.h>
29 #include <linux/list.h>
30 #include <linux/highmem.h>
31 #include <linux/pagevec.h>
37 #define NILFS_BUFFER_INHERENT_BITS \
38 ((1UL << BH_Uptodate) | (1UL << BH_Mapped) | (1UL << BH_NILFS_Node) | \
39 (1UL << BH_NILFS_Volatile) | (1UL << BH_NILFS_Allocated))
41 static struct buffer_head
*
42 __nilfs_get_page_block(struct page
*page
, unsigned long block
, pgoff_t index
,
43 int blkbits
, unsigned long b_state
)
46 unsigned long first_block
;
47 struct buffer_head
*bh
;
49 if (!page_has_buffers(page
))
50 create_empty_buffers(page
, 1 << blkbits
, b_state
);
52 first_block
= (unsigned long)index
<< (PAGE_CACHE_SHIFT
- blkbits
);
53 bh
= nilfs_page_get_nth_block(page
, block
- first_block
);
61 * Since the page cache of B-tree node pages or data page cache of pseudo
62 * inodes does not have a valid mapping->host pointer, calling
63 * mark_buffer_dirty() for their buffers causes a NULL pointer dereference;
64 * it calls __mark_inode_dirty(NULL) through __set_page_dirty().
65 * To avoid this problem, the old style mark_buffer_dirty() is used instead.
67 void nilfs_mark_buffer_dirty(struct buffer_head
*bh
)
69 if (!buffer_dirty(bh
) && !test_set_buffer_dirty(bh
))
70 __set_page_dirty_nobuffers(bh
->b_page
);
73 struct buffer_head
*nilfs_grab_buffer(struct inode
*inode
,
74 struct address_space
*mapping
,
76 unsigned long b_state
)
78 int blkbits
= inode
->i_blkbits
;
79 pgoff_t index
= blkoff
>> (PAGE_CACHE_SHIFT
- blkbits
);
80 struct page
*page
, *opage
;
81 struct buffer_head
*bh
, *obh
;
83 page
= grab_cache_page(mapping
, index
);
87 bh
= __nilfs_get_page_block(page
, blkoff
, index
, blkbits
, b_state
);
90 page_cache_release(page
);
93 if (!buffer_uptodate(bh
) && mapping
->assoc_mapping
!= NULL
) {
95 * Shadow page cache uses assoc_mapping to point its original
96 * page cache. The following code tries the original cache
97 * if the given cache is a shadow and it didn't hit.
99 opage
= find_lock_page(mapping
->assoc_mapping
, index
);
103 obh
= __nilfs_get_page_block(opage
, blkoff
, index
, blkbits
,
105 if (buffer_uptodate(obh
)) {
106 nilfs_copy_buffer(bh
, obh
);
107 if (buffer_dirty(obh
)) {
108 nilfs_mark_buffer_dirty(bh
);
109 if (!buffer_nilfs_node(bh
) && NILFS_MDT(inode
))
110 nilfs_mdt_mark_dirty(inode
);
115 page_cache_release(opage
);
121 * nilfs_forget_buffer - discard dirty state
122 * @inode: owner inode of the buffer
123 * @bh: buffer head of the buffer to be discarded
125 void nilfs_forget_buffer(struct buffer_head
*bh
)
127 struct page
*page
= bh
->b_page
;
130 clear_buffer_nilfs_volatile(bh
);
131 clear_buffer_dirty(bh
);
132 if (nilfs_page_buffers_clean(page
))
133 __nilfs_clear_page_dirty(page
);
135 clear_buffer_uptodate(bh
);
136 clear_buffer_mapped(bh
);
138 ClearPageUptodate(page
);
139 ClearPageMappedToDisk(page
);
145 * nilfs_copy_buffer -- copy buffer data and flags
146 * @dbh: destination buffer
147 * @sbh: source buffer
149 void nilfs_copy_buffer(struct buffer_head
*dbh
, struct buffer_head
*sbh
)
151 void *kaddr0
, *kaddr1
;
153 struct page
*spage
= sbh
->b_page
, *dpage
= dbh
->b_page
;
154 struct buffer_head
*bh
;
156 kaddr0
= kmap_atomic(spage
, KM_USER0
);
157 kaddr1
= kmap_atomic(dpage
, KM_USER1
);
158 memcpy(kaddr1
+ bh_offset(dbh
), kaddr0
+ bh_offset(sbh
), sbh
->b_size
);
159 kunmap_atomic(kaddr1
, KM_USER1
);
160 kunmap_atomic(kaddr0
, KM_USER0
);
162 dbh
->b_state
= sbh
->b_state
& NILFS_BUFFER_INHERENT_BITS
;
163 dbh
->b_blocknr
= sbh
->b_blocknr
;
164 dbh
->b_bdev
= sbh
->b_bdev
;
167 bits
= sbh
->b_state
& ((1UL << BH_Uptodate
) | (1UL << BH_Mapped
));
168 while ((bh
= bh
->b_this_page
) != dbh
) {
173 if (bits
& (1UL << BH_Uptodate
))
174 SetPageUptodate(dpage
);
176 ClearPageUptodate(dpage
);
177 if (bits
& (1UL << BH_Mapped
))
178 SetPageMappedToDisk(dpage
);
180 ClearPageMappedToDisk(dpage
);
184 * nilfs_page_buffers_clean - check if a page has dirty buffers or not.
185 * @page: page to be checked
187 * nilfs_page_buffers_clean() returns zero if the page has dirty buffers.
188 * Otherwise, it returns non-zero value.
190 int nilfs_page_buffers_clean(struct page
*page
)
192 struct buffer_head
*bh
, *head
;
194 bh
= head
= page_buffers(page
);
196 if (buffer_dirty(bh
))
198 bh
= bh
->b_this_page
;
199 } while (bh
!= head
);
203 void nilfs_page_bug(struct page
*page
)
205 struct address_space
*m
;
206 unsigned long ino
= 0;
208 if (unlikely(!page
)) {
209 printk(KERN_CRIT
"NILFS_PAGE_BUG(NULL)\n");
215 struct inode
*inode
= NILFS_AS_I(m
);
219 printk(KERN_CRIT
"NILFS_PAGE_BUG(%p): cnt=%d index#=%llu flags=0x%lx "
220 "mapping=%p ino=%lu\n",
221 page
, atomic_read(&page
->_count
),
222 (unsigned long long)page
->index
, page
->flags
, m
, ino
);
224 if (page_has_buffers(page
)) {
225 struct buffer_head
*bh
, *head
;
228 bh
= head
= page_buffers(page
);
231 " BH[%d] %p: cnt=%d block#=%llu state=0x%lx\n",
232 i
++, bh
, atomic_read(&bh
->b_count
),
233 (unsigned long long)bh
->b_blocknr
, bh
->b_state
);
234 bh
= bh
->b_this_page
;
235 } while (bh
!= head
);
240 * nilfs_alloc_private_page - allocate a private page with buffer heads
242 * Return Value: On success, a pointer to the allocated page is returned.
243 * On error, NULL is returned.
245 struct page
*nilfs_alloc_private_page(struct block_device
*bdev
, int size
,
248 struct buffer_head
*bh
, *head
, *tail
;
251 page
= alloc_page(GFP_NOFS
); /* page_count of the returned page is 1 */
256 head
= alloc_page_buffers(page
, size
, 0);
257 if (unlikely(!head
)) {
265 bh
->b_state
= (1UL << BH_NILFS_Allocated
) | state
;
268 bh
= bh
->b_this_page
;
271 tail
->b_this_page
= head
;
272 attach_page_buffers(page
, head
);
277 void nilfs_free_private_page(struct page
*page
)
279 BUG_ON(!PageLocked(page
));
280 BUG_ON(page
->mapping
);
282 if (page_has_buffers(page
) && !try_to_free_buffers(page
))
283 NILFS_PAGE_BUG(page
, "failed to free page");
290 * nilfs_copy_page -- copy the page with buffers
291 * @dst: destination page
293 * @copy_dirty: flag whether to copy dirty states on the page's buffer heads.
295 * This function is for both data pages and btnode pages. The dirty flag
296 * should be treated by caller. The page must not be under i/o.
297 * Both src and dst page must be locked
299 static void nilfs_copy_page(struct page
*dst
, struct page
*src
, int copy_dirty
)
301 struct buffer_head
*dbh
, *dbufs
, *sbh
, *sbufs
;
302 unsigned long mask
= NILFS_BUFFER_INHERENT_BITS
;
304 BUG_ON(PageWriteback(dst
));
306 sbh
= sbufs
= page_buffers(src
);
307 if (!page_has_buffers(dst
))
308 create_empty_buffers(dst
, sbh
->b_size
, 0);
311 mask
|= (1UL << BH_Dirty
);
313 dbh
= dbufs
= page_buffers(dst
);
317 dbh
->b_state
= sbh
->b_state
& mask
;
318 dbh
->b_blocknr
= sbh
->b_blocknr
;
319 dbh
->b_bdev
= sbh
->b_bdev
;
320 sbh
= sbh
->b_this_page
;
321 dbh
= dbh
->b_this_page
;
322 } while (dbh
!= dbufs
);
324 copy_highpage(dst
, src
);
326 if (PageUptodate(src
) && !PageUptodate(dst
))
327 SetPageUptodate(dst
);
328 else if (!PageUptodate(src
) && PageUptodate(dst
))
329 ClearPageUptodate(dst
);
330 if (PageMappedToDisk(src
) && !PageMappedToDisk(dst
))
331 SetPageMappedToDisk(dst
);
332 else if (!PageMappedToDisk(src
) && PageMappedToDisk(dst
))
333 ClearPageMappedToDisk(dst
);
338 sbh
= sbh
->b_this_page
;
339 dbh
= dbh
->b_this_page
;
340 } while (dbh
!= dbufs
);
343 int nilfs_copy_dirty_pages(struct address_space
*dmap
,
344 struct address_space
*smap
)
351 pagevec_init(&pvec
, 0);
353 if (!pagevec_lookup_tag(&pvec
, smap
, &index
, PAGECACHE_TAG_DIRTY
,
357 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
358 struct page
*page
= pvec
.pages
[i
], *dpage
;
361 if (unlikely(!PageDirty(page
)))
362 NILFS_PAGE_BUG(page
, "inconsistent dirty state");
364 dpage
= grab_cache_page(dmap
, page
->index
);
365 if (unlikely(!dpage
)) {
366 /* No empty page is added to the page cache */
371 if (unlikely(!page_has_buffers(page
)))
373 "found empty page in dat page cache");
375 nilfs_copy_page(dpage
, page
, 1);
376 __set_page_dirty_nobuffers(dpage
);
379 page_cache_release(dpage
);
382 pagevec_release(&pvec
);
391 * nilfs_copy_back_pages -- copy back pages to original cache from shadow cache
392 * @dmap: destination page cache
393 * @smap: source page cache
395 * No pages must no be added to the cache during this process.
396 * This must be ensured by the caller.
398 void nilfs_copy_back_pages(struct address_space
*dmap
,
399 struct address_space
*smap
)
406 pagevec_init(&pvec
, 0);
408 n
= pagevec_lookup(&pvec
, smap
, index
, PAGEVEC_SIZE
);
411 index
= pvec
.pages
[n
- 1]->index
+ 1;
413 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
414 struct page
*page
= pvec
.pages
[i
], *dpage
;
415 pgoff_t offset
= page
->index
;
418 dpage
= find_lock_page(dmap
, offset
);
420 /* override existing page on the destination cache */
421 WARN_ON(PageDirty(dpage
));
422 nilfs_copy_page(dpage
, page
, 0);
424 page_cache_release(dpage
);
428 /* move the page to the destination cache */
429 spin_lock_irq(&smap
->tree_lock
);
430 page2
= radix_tree_delete(&smap
->page_tree
, offset
);
431 WARN_ON(page2
!= page
);
434 spin_unlock_irq(&smap
->tree_lock
);
436 spin_lock_irq(&dmap
->tree_lock
);
437 err
= radix_tree_insert(&dmap
->page_tree
, offset
, page
);
438 if (unlikely(err
< 0)) {
439 WARN_ON(err
== -EEXIST
);
440 page
->mapping
= NULL
;
441 page_cache_release(page
); /* for cache */
443 page
->mapping
= dmap
;
446 radix_tree_tag_set(&dmap
->page_tree
,
448 PAGECACHE_TAG_DIRTY
);
450 spin_unlock_irq(&dmap
->tree_lock
);
454 pagevec_release(&pvec
);
460 void nilfs_clear_dirty_pages(struct address_space
*mapping
)
466 pagevec_init(&pvec
, 0);
468 while (pagevec_lookup_tag(&pvec
, mapping
, &index
, PAGECACHE_TAG_DIRTY
,
470 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
471 struct page
*page
= pvec
.pages
[i
];
472 struct buffer_head
*bh
, *head
;
475 ClearPageUptodate(page
);
476 ClearPageMappedToDisk(page
);
477 bh
= head
= page_buffers(page
);
480 clear_buffer_dirty(bh
);
481 clear_buffer_nilfs_volatile(bh
);
482 clear_buffer_uptodate(bh
);
483 clear_buffer_mapped(bh
);
485 bh
= bh
->b_this_page
;
486 } while (bh
!= head
);
488 __nilfs_clear_page_dirty(page
);
491 pagevec_release(&pvec
);
496 unsigned nilfs_page_count_clean_buffers(struct page
*page
,
497 unsigned from
, unsigned to
)
499 unsigned block_start
, block_end
;
500 struct buffer_head
*bh
, *head
;
503 for (bh
= head
= page_buffers(page
), block_start
= 0;
504 bh
!= head
|| !block_start
;
505 block_start
= block_end
, bh
= bh
->b_this_page
) {
506 block_end
= block_start
+ bh
->b_size
;
507 if (block_end
> from
&& block_start
< to
&& !buffer_dirty(bh
))
514 * NILFS2 needs clear_page_dirty() in the following two cases:
516 * 1) For B-tree node pages and data pages of the dat/gcdat, NILFS2 clears
517 * page dirty flags when it copies back pages from the shadow cache
518 * (gcdat->{i_mapping,i_btnode_cache}) to its original cache
519 * (dat->{i_mapping,i_btnode_cache}).
521 * 2) Some B-tree operations like insertion or deletion may dispose buffers
522 * in dirty state, and this needs to cancel the dirty state of their pages.
524 int __nilfs_clear_page_dirty(struct page
*page
)
526 struct address_space
*mapping
= page
->mapping
;
529 spin_lock_irq(&mapping
->tree_lock
);
530 if (test_bit(PG_dirty
, &page
->flags
)) {
531 radix_tree_tag_clear(&mapping
->page_tree
,
533 PAGECACHE_TAG_DIRTY
);
534 spin_unlock_irq(&mapping
->tree_lock
);
535 return clear_page_dirty_for_io(page
);
537 spin_unlock_irq(&mapping
->tree_lock
);
540 return TestClearPageDirty(page
);