4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/f2fs_fs.h>
13 #include <linux/bio.h>
14 #include <linux/blkdev.h>
15 #include <linux/vmalloc.h>
21 static int need_to_flush(struct f2fs_sb_info
*sbi
)
23 unsigned int pages_per_sec
= (1 << sbi
->log_blocks_per_seg
) *
25 int node_secs
= ((get_pages(sbi
, F2FS_DIRTY_NODES
) + pages_per_sec
- 1)
26 >> sbi
->log_blocks_per_seg
) / sbi
->segs_per_sec
;
27 int dent_secs
= ((get_pages(sbi
, F2FS_DIRTY_DENTS
) + pages_per_sec
- 1)
28 >> sbi
->log_blocks_per_seg
) / sbi
->segs_per_sec
;
33 if (free_sections(sbi
) <= (node_secs
+ 2 * dent_secs
+
34 reserved_sections(sbi
)))
40 * This function balances dirty node and dentry pages.
41 * In addition, it controls garbage collection.
43 void f2fs_balance_fs(struct f2fs_sb_info
*sbi
)
45 struct writeback_control wbc
= {
46 .sync_mode
= WB_SYNC_ALL
,
47 .nr_to_write
= LONG_MAX
,
55 * We should do checkpoint when there are so many dirty node pages
56 * with enough free segments. After then, we should do GC.
58 if (need_to_flush(sbi
)) {
59 sync_dirty_dir_inodes(sbi
);
60 sync_node_pages(sbi
, 0, &wbc
);
63 if (has_not_enough_free_secs(sbi
)) {
64 mutex_lock(&sbi
->gc_mutex
);
69 static void __locate_dirty_segment(struct f2fs_sb_info
*sbi
, unsigned int segno
,
70 enum dirty_type dirty_type
)
72 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
74 /* need not be added */
75 if (IS_CURSEG(sbi
, segno
))
78 if (!test_and_set_bit(segno
, dirty_i
->dirty_segmap
[dirty_type
]))
79 dirty_i
->nr_dirty
[dirty_type
]++;
81 if (dirty_type
== DIRTY
) {
82 struct seg_entry
*sentry
= get_seg_entry(sbi
, segno
);
83 dirty_type
= sentry
->type
;
84 if (!test_and_set_bit(segno
, dirty_i
->dirty_segmap
[dirty_type
]))
85 dirty_i
->nr_dirty
[dirty_type
]++;
89 static void __remove_dirty_segment(struct f2fs_sb_info
*sbi
, unsigned int segno
,
90 enum dirty_type dirty_type
)
92 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
94 if (test_and_clear_bit(segno
, dirty_i
->dirty_segmap
[dirty_type
]))
95 dirty_i
->nr_dirty
[dirty_type
]--;
97 if (dirty_type
== DIRTY
) {
98 struct seg_entry
*sentry
= get_seg_entry(sbi
, segno
);
99 dirty_type
= sentry
->type
;
100 if (test_and_clear_bit(segno
,
101 dirty_i
->dirty_segmap
[dirty_type
]))
102 dirty_i
->nr_dirty
[dirty_type
]--;
103 clear_bit(segno
, dirty_i
->victim_segmap
[FG_GC
]);
104 clear_bit(segno
, dirty_i
->victim_segmap
[BG_GC
]);
109 * Should not occur error such as -ENOMEM.
110 * Adding dirty entry into seglist is not critical operation.
111 * If a given segment is one of current working segments, it won't be added.
113 void locate_dirty_segment(struct f2fs_sb_info
*sbi
, unsigned int segno
)
115 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
116 unsigned short valid_blocks
;
118 if (segno
== NULL_SEGNO
|| IS_CURSEG(sbi
, segno
))
121 mutex_lock(&dirty_i
->seglist_lock
);
123 valid_blocks
= get_valid_blocks(sbi
, segno
, 0);
125 if (valid_blocks
== 0) {
126 __locate_dirty_segment(sbi
, segno
, PRE
);
127 __remove_dirty_segment(sbi
, segno
, DIRTY
);
128 } else if (valid_blocks
< sbi
->blocks_per_seg
) {
129 __locate_dirty_segment(sbi
, segno
, DIRTY
);
131 /* Recovery routine with SSR needs this */
132 __remove_dirty_segment(sbi
, segno
, DIRTY
);
135 mutex_unlock(&dirty_i
->seglist_lock
);
140 * Should call clear_prefree_segments after checkpoint is done.
142 static void set_prefree_as_free_segments(struct f2fs_sb_info
*sbi
)
144 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
145 unsigned int segno
, offset
= 0;
146 unsigned int total_segs
= TOTAL_SEGS(sbi
);
148 mutex_lock(&dirty_i
->seglist_lock
);
150 segno
= find_next_bit(dirty_i
->dirty_segmap
[PRE
], total_segs
,
152 if (segno
>= total_segs
)
154 __set_test_and_free(sbi
, segno
);
157 mutex_unlock(&dirty_i
->seglist_lock
);
160 void clear_prefree_segments(struct f2fs_sb_info
*sbi
)
162 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
163 unsigned int segno
, offset
= 0;
164 unsigned int total_segs
= TOTAL_SEGS(sbi
);
166 mutex_lock(&dirty_i
->seglist_lock
);
168 segno
= find_next_bit(dirty_i
->dirty_segmap
[PRE
], total_segs
,
170 if (segno
>= total_segs
)
174 if (test_and_clear_bit(segno
, dirty_i
->dirty_segmap
[PRE
]))
175 dirty_i
->nr_dirty
[PRE
]--;
178 if (test_opt(sbi
, DISCARD
))
179 blkdev_issue_discard(sbi
->sb
->s_bdev
,
180 START_BLOCK(sbi
, segno
) <<
181 sbi
->log_sectors_per_block
,
182 1 << (sbi
->log_sectors_per_block
+
183 sbi
->log_blocks_per_seg
),
186 mutex_unlock(&dirty_i
->seglist_lock
);
189 static void __mark_sit_entry_dirty(struct f2fs_sb_info
*sbi
, unsigned int segno
)
191 struct sit_info
*sit_i
= SIT_I(sbi
);
192 if (!__test_and_set_bit(segno
, sit_i
->dirty_sentries_bitmap
))
193 sit_i
->dirty_sentries
++;
196 static void __set_sit_entry_type(struct f2fs_sb_info
*sbi
, int type
,
197 unsigned int segno
, int modified
)
199 struct seg_entry
*se
= get_seg_entry(sbi
, segno
);
202 __mark_sit_entry_dirty(sbi
, segno
);
205 static void update_sit_entry(struct f2fs_sb_info
*sbi
, block_t blkaddr
, int del
)
207 struct seg_entry
*se
;
208 unsigned int segno
, offset
;
209 long int new_vblocks
;
211 segno
= GET_SEGNO(sbi
, blkaddr
);
213 se
= get_seg_entry(sbi
, segno
);
214 new_vblocks
= se
->valid_blocks
+ del
;
215 offset
= GET_SEGOFF_FROM_SEG0(sbi
, blkaddr
) & (sbi
->blocks_per_seg
- 1);
217 BUG_ON((new_vblocks
>> (sizeof(unsigned short) << 3) ||
218 (new_vblocks
> sbi
->blocks_per_seg
)));
220 se
->valid_blocks
= new_vblocks
;
221 se
->mtime
= get_mtime(sbi
);
222 SIT_I(sbi
)->max_mtime
= se
->mtime
;
224 /* Update valid block bitmap */
226 if (f2fs_set_bit(offset
, se
->cur_valid_map
))
229 if (!f2fs_clear_bit(offset
, se
->cur_valid_map
))
232 if (!f2fs_test_bit(offset
, se
->ckpt_valid_map
))
233 se
->ckpt_valid_blocks
+= del
;
235 __mark_sit_entry_dirty(sbi
, segno
);
237 /* update total number of valid blocks to be written in ckpt area */
238 SIT_I(sbi
)->written_valid_blocks
+= del
;
240 if (sbi
->segs_per_sec
> 1)
241 get_sec_entry(sbi
, segno
)->valid_blocks
+= del
;
244 static void refresh_sit_entry(struct f2fs_sb_info
*sbi
,
245 block_t old_blkaddr
, block_t new_blkaddr
)
247 update_sit_entry(sbi
, new_blkaddr
, 1);
248 if (GET_SEGNO(sbi
, old_blkaddr
) != NULL_SEGNO
)
249 update_sit_entry(sbi
, old_blkaddr
, -1);
252 void invalidate_blocks(struct f2fs_sb_info
*sbi
, block_t addr
)
254 unsigned int segno
= GET_SEGNO(sbi
, addr
);
255 struct sit_info
*sit_i
= SIT_I(sbi
);
257 BUG_ON(addr
== NULL_ADDR
);
258 if (addr
== NEW_ADDR
)
261 /* add it into sit main buffer */
262 mutex_lock(&sit_i
->sentry_lock
);
264 update_sit_entry(sbi
, addr
, -1);
266 /* add it into dirty seglist */
267 locate_dirty_segment(sbi
, segno
);
269 mutex_unlock(&sit_i
->sentry_lock
);
273 * This function should be resided under the curseg_mutex lock
275 static void __add_sum_entry(struct f2fs_sb_info
*sbi
, int type
,
276 struct f2fs_summary
*sum
, unsigned short offset
)
278 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
279 void *addr
= curseg
->sum_blk
;
280 addr
+= offset
* sizeof(struct f2fs_summary
);
281 memcpy(addr
, sum
, sizeof(struct f2fs_summary
));
286 * Calculate the number of current summary pages for writing
288 int npages_for_summary_flush(struct f2fs_sb_info
*sbi
)
290 int total_size_bytes
= 0;
291 int valid_sum_count
= 0;
294 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
295 if (sbi
->ckpt
->alloc_type
[i
] == SSR
)
296 valid_sum_count
+= sbi
->blocks_per_seg
;
298 valid_sum_count
+= curseg_blkoff(sbi
, i
);
301 total_size_bytes
= valid_sum_count
* (SUMMARY_SIZE
+ 1)
302 + sizeof(struct nat_journal
) + 2
303 + sizeof(struct sit_journal
) + 2;
304 sum_space
= PAGE_CACHE_SIZE
- SUM_FOOTER_SIZE
;
305 if (total_size_bytes
< sum_space
)
307 else if (total_size_bytes
< 2 * sum_space
)
313 * Caller should put this summary page
315 struct page
*get_sum_page(struct f2fs_sb_info
*sbi
, unsigned int segno
)
317 return get_meta_page(sbi
, GET_SUM_BLOCK(sbi
, segno
));
320 static void write_sum_page(struct f2fs_sb_info
*sbi
,
321 struct f2fs_summary_block
*sum_blk
, block_t blk_addr
)
323 struct page
*page
= grab_meta_page(sbi
, blk_addr
);
324 void *kaddr
= page_address(page
);
325 memcpy(kaddr
, sum_blk
, PAGE_CACHE_SIZE
);
326 set_page_dirty(page
);
327 f2fs_put_page(page
, 1);
330 static unsigned int check_prefree_segments(struct f2fs_sb_info
*sbi
,
331 int ofs_unit
, int type
)
333 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
334 unsigned long *prefree_segmap
= dirty_i
->dirty_segmap
[PRE
];
335 unsigned int segno
, next_segno
, i
;
339 * If there is not enough reserved sections,
340 * we should not reuse prefree segments.
342 if (has_not_enough_free_secs(sbi
))
346 * NODE page should not reuse prefree segment,
347 * since those information is used for SPOR.
349 if (IS_NODESEG(type
))
352 segno
= find_next_bit(prefree_segmap
, TOTAL_SEGS(sbi
), ofs
++);
353 ofs
= ((segno
/ ofs_unit
) * ofs_unit
) + ofs_unit
;
354 if (segno
< TOTAL_SEGS(sbi
)) {
355 /* skip intermediate segments in a section */
356 if (segno
% ofs_unit
)
359 /* skip if whole section is not prefree */
360 next_segno
= find_next_zero_bit(prefree_segmap
,
361 TOTAL_SEGS(sbi
), segno
+ 1);
362 if (next_segno
- segno
< ofs_unit
)
365 /* skip if whole section was not free at the last checkpoint */
366 for (i
= 0; i
< ofs_unit
; i
++)
367 if (get_seg_entry(sbi
, segno
)->ckpt_valid_blocks
)
375 * Find a new segment from the free segments bitmap to right order
376 * This function should be returned with success, otherwise BUG
378 static void get_new_segment(struct f2fs_sb_info
*sbi
,
379 unsigned int *newseg
, bool new_sec
, int dir
)
381 struct free_segmap_info
*free_i
= FREE_I(sbi
);
382 unsigned int total_secs
= sbi
->total_sections
;
383 unsigned int segno
, secno
, zoneno
;
384 unsigned int total_zones
= sbi
->total_sections
/ sbi
->secs_per_zone
;
385 unsigned int hint
= *newseg
/ sbi
->segs_per_sec
;
386 unsigned int old_zoneno
= GET_ZONENO_FROM_SEGNO(sbi
, *newseg
);
387 unsigned int left_start
= hint
;
392 write_lock(&free_i
->segmap_lock
);
394 if (!new_sec
&& ((*newseg
+ 1) % sbi
->segs_per_sec
)) {
395 segno
= find_next_zero_bit(free_i
->free_segmap
,
396 TOTAL_SEGS(sbi
), *newseg
+ 1);
397 if (segno
< TOTAL_SEGS(sbi
))
401 secno
= find_next_zero_bit(free_i
->free_secmap
, total_secs
, hint
);
402 if (secno
>= total_secs
) {
403 if (dir
== ALLOC_RIGHT
) {
404 secno
= find_next_zero_bit(free_i
->free_secmap
,
406 BUG_ON(secno
>= total_secs
);
409 left_start
= hint
- 1;
415 while (test_bit(left_start
, free_i
->free_secmap
)) {
416 if (left_start
> 0) {
420 left_start
= find_next_zero_bit(free_i
->free_secmap
,
422 BUG_ON(left_start
>= total_secs
);
428 segno
= secno
* sbi
->segs_per_sec
;
429 zoneno
= secno
/ sbi
->secs_per_zone
;
431 /* give up on finding another zone */
434 if (sbi
->secs_per_zone
== 1)
436 if (zoneno
== old_zoneno
)
438 if (dir
== ALLOC_LEFT
) {
439 if (!go_left
&& zoneno
+ 1 >= total_zones
)
441 if (go_left
&& zoneno
== 0)
444 for (i
= 0; i
< NR_CURSEG_TYPE
; i
++)
445 if (CURSEG_I(sbi
, i
)->zone
== zoneno
)
448 if (i
< NR_CURSEG_TYPE
) {
449 /* zone is in user, try another */
451 hint
= zoneno
* sbi
->secs_per_zone
- 1;
452 else if (zoneno
+ 1 >= total_zones
)
455 hint
= (zoneno
+ 1) * sbi
->secs_per_zone
;
457 goto find_other_zone
;
460 /* set it as dirty segment in free segmap */
461 BUG_ON(test_bit(segno
, free_i
->free_segmap
));
462 __set_inuse(sbi
, segno
);
464 write_unlock(&free_i
->segmap_lock
);
467 static void reset_curseg(struct f2fs_sb_info
*sbi
, int type
, int modified
)
469 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
470 struct summary_footer
*sum_footer
;
472 curseg
->segno
= curseg
->next_segno
;
473 curseg
->zone
= GET_ZONENO_FROM_SEGNO(sbi
, curseg
->segno
);
474 curseg
->next_blkoff
= 0;
475 curseg
->next_segno
= NULL_SEGNO
;
477 sum_footer
= &(curseg
->sum_blk
->footer
);
478 memset(sum_footer
, 0, sizeof(struct summary_footer
));
479 if (IS_DATASEG(type
))
480 SET_SUM_TYPE(sum_footer
, SUM_TYPE_DATA
);
481 if (IS_NODESEG(type
))
482 SET_SUM_TYPE(sum_footer
, SUM_TYPE_NODE
);
483 __set_sit_entry_type(sbi
, type
, curseg
->segno
, modified
);
487 * Allocate a current working segment.
488 * This function always allocates a free segment in LFS manner.
490 static void new_curseg(struct f2fs_sb_info
*sbi
, int type
, bool new_sec
)
492 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
493 unsigned int segno
= curseg
->segno
;
494 int dir
= ALLOC_LEFT
;
496 write_sum_page(sbi
, curseg
->sum_blk
,
497 GET_SUM_BLOCK(sbi
, curseg
->segno
));
498 if (type
== CURSEG_WARM_DATA
|| type
== CURSEG_COLD_DATA
)
501 if (test_opt(sbi
, NOHEAP
))
504 get_new_segment(sbi
, &segno
, new_sec
, dir
);
505 curseg
->next_segno
= segno
;
506 reset_curseg(sbi
, type
, 1);
507 curseg
->alloc_type
= LFS
;
510 static void __next_free_blkoff(struct f2fs_sb_info
*sbi
,
511 struct curseg_info
*seg
, block_t start
)
513 struct seg_entry
*se
= get_seg_entry(sbi
, seg
->segno
);
515 for (ofs
= start
; ofs
< sbi
->blocks_per_seg
; ofs
++) {
516 if (!f2fs_test_bit(ofs
, se
->ckpt_valid_map
)
517 && !f2fs_test_bit(ofs
, se
->cur_valid_map
))
520 seg
->next_blkoff
= ofs
;
524 * If a segment is written by LFS manner, next block offset is just obtained
525 * by increasing the current block offset. However, if a segment is written by
526 * SSR manner, next block offset obtained by calling __next_free_blkoff
528 static void __refresh_next_blkoff(struct f2fs_sb_info
*sbi
,
529 struct curseg_info
*seg
)
531 if (seg
->alloc_type
== SSR
)
532 __next_free_blkoff(sbi
, seg
, seg
->next_blkoff
+ 1);
538 * This function always allocates a used segment (from dirty seglist) by SSR
539 * manner, so it should recover the existing segment information of valid blocks
541 static void change_curseg(struct f2fs_sb_info
*sbi
, int type
, bool reuse
)
543 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
544 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
545 unsigned int new_segno
= curseg
->next_segno
;
546 struct f2fs_summary_block
*sum_node
;
547 struct page
*sum_page
;
549 write_sum_page(sbi
, curseg
->sum_blk
,
550 GET_SUM_BLOCK(sbi
, curseg
->segno
));
551 __set_test_and_inuse(sbi
, new_segno
);
553 mutex_lock(&dirty_i
->seglist_lock
);
554 __remove_dirty_segment(sbi
, new_segno
, PRE
);
555 __remove_dirty_segment(sbi
, new_segno
, DIRTY
);
556 mutex_unlock(&dirty_i
->seglist_lock
);
558 reset_curseg(sbi
, type
, 1);
559 curseg
->alloc_type
= SSR
;
560 __next_free_blkoff(sbi
, curseg
, 0);
563 sum_page
= get_sum_page(sbi
, new_segno
);
564 sum_node
= (struct f2fs_summary_block
*)page_address(sum_page
);
565 memcpy(curseg
->sum_blk
, sum_node
, SUM_ENTRY_SIZE
);
566 f2fs_put_page(sum_page
, 1);
571 * flush out current segment and replace it with new segment
572 * This function should be returned with success, otherwise BUG
574 static void allocate_segment_by_default(struct f2fs_sb_info
*sbi
,
575 int type
, bool force
)
577 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
578 unsigned int ofs_unit
;
581 new_curseg(sbi
, type
, true);
585 ofs_unit
= need_SSR(sbi
) ? 1 : sbi
->segs_per_sec
;
586 curseg
->next_segno
= check_prefree_segments(sbi
, ofs_unit
, type
);
588 if (curseg
->next_segno
!= NULL_SEGNO
)
589 change_curseg(sbi
, type
, false);
590 else if (type
== CURSEG_WARM_NODE
)
591 new_curseg(sbi
, type
, false);
592 else if (need_SSR(sbi
) && get_ssr_segment(sbi
, type
))
593 change_curseg(sbi
, type
, true);
595 new_curseg(sbi
, type
, false);
597 sbi
->segment_count
[curseg
->alloc_type
]++;
600 void allocate_new_segments(struct f2fs_sb_info
*sbi
)
602 struct curseg_info
*curseg
;
603 unsigned int old_curseg
;
606 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
607 curseg
= CURSEG_I(sbi
, i
);
608 old_curseg
= curseg
->segno
;
609 SIT_I(sbi
)->s_ops
->allocate_segment(sbi
, i
, true);
610 locate_dirty_segment(sbi
, old_curseg
);
614 static const struct segment_allocation default_salloc_ops
= {
615 .allocate_segment
= allocate_segment_by_default
,
618 static void f2fs_end_io_write(struct bio
*bio
, int err
)
620 const int uptodate
= test_bit(BIO_UPTODATE
, &bio
->bi_flags
);
621 struct bio_vec
*bvec
= bio
->bi_io_vec
+ bio
->bi_vcnt
- 1;
622 struct bio_private
*p
= bio
->bi_private
;
625 struct page
*page
= bvec
->bv_page
;
627 if (--bvec
>= bio
->bi_io_vec
)
628 prefetchw(&bvec
->bv_page
->flags
);
632 set_bit(AS_EIO
, &page
->mapping
->flags
);
633 set_ckpt_flags(p
->sbi
->ckpt
, CP_ERROR_FLAG
);
634 set_page_dirty(page
);
636 end_page_writeback(page
);
637 dec_page_count(p
->sbi
, F2FS_WRITEBACK
);
638 } while (bvec
>= bio
->bi_io_vec
);
646 struct bio
*f2fs_bio_alloc(struct block_device
*bdev
, sector_t first_sector
,
647 int nr_vecs
, gfp_t gfp_flags
)
651 /* allocate new bio */
652 bio
= bio_alloc(gfp_flags
, nr_vecs
);
654 if (bio
== NULL
&& (current
->flags
& PF_MEMALLOC
)) {
655 while (!bio
&& (nr_vecs
/= 2))
656 bio
= bio_alloc(gfp_flags
, nr_vecs
);
660 bio
->bi_sector
= first_sector
;
662 bio
->bi_private
= kmalloc(sizeof(struct bio_private
),
663 GFP_NOFS
| __GFP_HIGH
);
664 if (!bio
->bi_private
) {
676 static void do_submit_bio(struct f2fs_sb_info
*sbi
,
677 enum page_type type
, bool sync
)
679 int rw
= sync
? WRITE_SYNC
: WRITE
;
680 enum page_type btype
= type
> META
? META
: type
;
682 if (type
>= META_FLUSH
)
683 rw
= WRITE_FLUSH_FUA
;
685 if (sbi
->bio
[btype
]) {
686 struct bio_private
*p
= sbi
->bio
[btype
]->bi_private
;
688 sbi
->bio
[btype
]->bi_end_io
= f2fs_end_io_write
;
689 if (type
== META_FLUSH
) {
690 DECLARE_COMPLETION_ONSTACK(wait
);
693 submit_bio(rw
, sbi
->bio
[btype
]);
694 wait_for_completion(&wait
);
697 submit_bio(rw
, sbi
->bio
[btype
]);
699 sbi
->bio
[btype
] = NULL
;
703 void f2fs_submit_bio(struct f2fs_sb_info
*sbi
, enum page_type type
, bool sync
)
705 down_write(&sbi
->bio_sem
);
706 do_submit_bio(sbi
, type
, sync
);
707 up_write(&sbi
->bio_sem
);
710 static void submit_write_page(struct f2fs_sb_info
*sbi
, struct page
*page
,
711 block_t blk_addr
, enum page_type type
)
713 struct block_device
*bdev
= sbi
->sb
->s_bdev
;
715 verify_block_addr(sbi
, blk_addr
);
717 down_write(&sbi
->bio_sem
);
719 inc_page_count(sbi
, F2FS_WRITEBACK
);
721 if (sbi
->bio
[type
] && sbi
->last_block_in_bio
[type
] != blk_addr
- 1)
722 do_submit_bio(sbi
, type
, false);
724 if (sbi
->bio
[type
] == NULL
)
725 sbi
->bio
[type
] = f2fs_bio_alloc(bdev
,
726 blk_addr
<< (sbi
->log_blocksize
- 9),
727 bio_get_nr_vecs(bdev
), GFP_NOFS
| __GFP_HIGH
);
729 if (bio_add_page(sbi
->bio
[type
], page
, PAGE_CACHE_SIZE
, 0) <
731 do_submit_bio(sbi
, type
, false);
735 sbi
->last_block_in_bio
[type
] = blk_addr
;
737 up_write(&sbi
->bio_sem
);
740 static bool __has_curseg_space(struct f2fs_sb_info
*sbi
, int type
)
742 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
743 if (curseg
->next_blkoff
< sbi
->blocks_per_seg
)
748 static int __get_segment_type_2(struct page
*page
, enum page_type p_type
)
751 return CURSEG_HOT_DATA
;
753 return CURSEG_HOT_NODE
;
756 static int __get_segment_type_4(struct page
*page
, enum page_type p_type
)
758 if (p_type
== DATA
) {
759 struct inode
*inode
= page
->mapping
->host
;
761 if (S_ISDIR(inode
->i_mode
))
762 return CURSEG_HOT_DATA
;
764 return CURSEG_COLD_DATA
;
766 if (IS_DNODE(page
) && !is_cold_node(page
))
767 return CURSEG_HOT_NODE
;
769 return CURSEG_COLD_NODE
;
773 static int __get_segment_type_6(struct page
*page
, enum page_type p_type
)
775 if (p_type
== DATA
) {
776 struct inode
*inode
= page
->mapping
->host
;
778 if (S_ISDIR(inode
->i_mode
))
779 return CURSEG_HOT_DATA
;
780 else if (is_cold_data(page
) || is_cold_file(inode
))
781 return CURSEG_COLD_DATA
;
783 return CURSEG_WARM_DATA
;
786 return is_cold_node(page
) ? CURSEG_WARM_NODE
:
789 return CURSEG_COLD_NODE
;
793 static int __get_segment_type(struct page
*page
, enum page_type p_type
)
795 struct f2fs_sb_info
*sbi
= F2FS_SB(page
->mapping
->host
->i_sb
);
796 switch (sbi
->active_logs
) {
798 return __get_segment_type_2(page
, p_type
);
800 return __get_segment_type_4(page
, p_type
);
802 return __get_segment_type_6(page
, p_type
);
808 static void do_write_page(struct f2fs_sb_info
*sbi
, struct page
*page
,
809 block_t old_blkaddr
, block_t
*new_blkaddr
,
810 struct f2fs_summary
*sum
, enum page_type p_type
)
812 struct sit_info
*sit_i
= SIT_I(sbi
);
813 struct curseg_info
*curseg
;
814 unsigned int old_cursegno
;
817 type
= __get_segment_type(page
, p_type
);
818 curseg
= CURSEG_I(sbi
, type
);
820 mutex_lock(&curseg
->curseg_mutex
);
822 *new_blkaddr
= NEXT_FREE_BLKADDR(sbi
, curseg
);
823 old_cursegno
= curseg
->segno
;
826 * __add_sum_entry should be resided under the curseg_mutex
827 * because, this function updates a summary entry in the
828 * current summary block.
830 __add_sum_entry(sbi
, type
, sum
, curseg
->next_blkoff
);
832 mutex_lock(&sit_i
->sentry_lock
);
833 __refresh_next_blkoff(sbi
, curseg
);
834 sbi
->block_count
[curseg
->alloc_type
]++;
837 * SIT information should be updated before segment allocation,
838 * since SSR needs latest valid block information.
840 refresh_sit_entry(sbi
, old_blkaddr
, *new_blkaddr
);
842 if (!__has_curseg_space(sbi
, type
))
843 sit_i
->s_ops
->allocate_segment(sbi
, type
, false);
845 locate_dirty_segment(sbi
, old_cursegno
);
846 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, old_blkaddr
));
847 mutex_unlock(&sit_i
->sentry_lock
);
850 fill_node_footer_blkaddr(page
, NEXT_FREE_BLKADDR(sbi
, curseg
));
852 /* writeout dirty page into bdev */
853 submit_write_page(sbi
, page
, *new_blkaddr
, p_type
);
855 mutex_unlock(&curseg
->curseg_mutex
);
858 int write_meta_page(struct f2fs_sb_info
*sbi
, struct page
*page
,
859 struct writeback_control
*wbc
)
861 if (wbc
->for_reclaim
)
862 return AOP_WRITEPAGE_ACTIVATE
;
864 set_page_writeback(page
);
865 submit_write_page(sbi
, page
, page
->index
, META
);
869 void write_node_page(struct f2fs_sb_info
*sbi
, struct page
*page
,
870 unsigned int nid
, block_t old_blkaddr
, block_t
*new_blkaddr
)
872 struct f2fs_summary sum
;
873 set_summary(&sum
, nid
, 0, 0);
874 do_write_page(sbi
, page
, old_blkaddr
, new_blkaddr
, &sum
, NODE
);
877 void write_data_page(struct inode
*inode
, struct page
*page
,
878 struct dnode_of_data
*dn
, block_t old_blkaddr
,
879 block_t
*new_blkaddr
)
881 struct f2fs_sb_info
*sbi
= F2FS_SB(inode
->i_sb
);
882 struct f2fs_summary sum
;
885 BUG_ON(old_blkaddr
== NULL_ADDR
);
886 get_node_info(sbi
, dn
->nid
, &ni
);
887 set_summary(&sum
, dn
->nid
, dn
->ofs_in_node
, ni
.version
);
889 do_write_page(sbi
, page
, old_blkaddr
,
890 new_blkaddr
, &sum
, DATA
);
893 void rewrite_data_page(struct f2fs_sb_info
*sbi
, struct page
*page
,
894 block_t old_blk_addr
)
896 submit_write_page(sbi
, page
, old_blk_addr
, DATA
);
899 void recover_data_page(struct f2fs_sb_info
*sbi
,
900 struct page
*page
, struct f2fs_summary
*sum
,
901 block_t old_blkaddr
, block_t new_blkaddr
)
903 struct sit_info
*sit_i
= SIT_I(sbi
);
904 struct curseg_info
*curseg
;
905 unsigned int segno
, old_cursegno
;
906 struct seg_entry
*se
;
909 segno
= GET_SEGNO(sbi
, new_blkaddr
);
910 se
= get_seg_entry(sbi
, segno
);
913 if (se
->valid_blocks
== 0 && !IS_CURSEG(sbi
, segno
)) {
914 if (old_blkaddr
== NULL_ADDR
)
915 type
= CURSEG_COLD_DATA
;
917 type
= CURSEG_WARM_DATA
;
919 curseg
= CURSEG_I(sbi
, type
);
921 mutex_lock(&curseg
->curseg_mutex
);
922 mutex_lock(&sit_i
->sentry_lock
);
924 old_cursegno
= curseg
->segno
;
926 /* change the current segment */
927 if (segno
!= curseg
->segno
) {
928 curseg
->next_segno
= segno
;
929 change_curseg(sbi
, type
, true);
932 curseg
->next_blkoff
= GET_SEGOFF_FROM_SEG0(sbi
, new_blkaddr
) &
933 (sbi
->blocks_per_seg
- 1);
934 __add_sum_entry(sbi
, type
, sum
, curseg
->next_blkoff
);
936 refresh_sit_entry(sbi
, old_blkaddr
, new_blkaddr
);
938 locate_dirty_segment(sbi
, old_cursegno
);
939 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, old_blkaddr
));
941 mutex_unlock(&sit_i
->sentry_lock
);
942 mutex_unlock(&curseg
->curseg_mutex
);
945 void rewrite_node_page(struct f2fs_sb_info
*sbi
,
946 struct page
*page
, struct f2fs_summary
*sum
,
947 block_t old_blkaddr
, block_t new_blkaddr
)
949 struct sit_info
*sit_i
= SIT_I(sbi
);
950 int type
= CURSEG_WARM_NODE
;
951 struct curseg_info
*curseg
;
952 unsigned int segno
, old_cursegno
;
953 block_t next_blkaddr
= next_blkaddr_of_node(page
);
954 unsigned int next_segno
= GET_SEGNO(sbi
, next_blkaddr
);
956 curseg
= CURSEG_I(sbi
, type
);
958 mutex_lock(&curseg
->curseg_mutex
);
959 mutex_lock(&sit_i
->sentry_lock
);
961 segno
= GET_SEGNO(sbi
, new_blkaddr
);
962 old_cursegno
= curseg
->segno
;
964 /* change the current segment */
965 if (segno
!= curseg
->segno
) {
966 curseg
->next_segno
= segno
;
967 change_curseg(sbi
, type
, true);
969 curseg
->next_blkoff
= GET_SEGOFF_FROM_SEG0(sbi
, new_blkaddr
) &
970 (sbi
->blocks_per_seg
- 1);
971 __add_sum_entry(sbi
, type
, sum
, curseg
->next_blkoff
);
973 /* change the current log to the next block addr in advance */
974 if (next_segno
!= segno
) {
975 curseg
->next_segno
= next_segno
;
976 change_curseg(sbi
, type
, true);
978 curseg
->next_blkoff
= GET_SEGOFF_FROM_SEG0(sbi
, next_blkaddr
) &
979 (sbi
->blocks_per_seg
- 1);
981 /* rewrite node page */
982 set_page_writeback(page
);
983 submit_write_page(sbi
, page
, new_blkaddr
, NODE
);
984 f2fs_submit_bio(sbi
, NODE
, true);
985 refresh_sit_entry(sbi
, old_blkaddr
, new_blkaddr
);
987 locate_dirty_segment(sbi
, old_cursegno
);
988 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, old_blkaddr
));
990 mutex_unlock(&sit_i
->sentry_lock
);
991 mutex_unlock(&curseg
->curseg_mutex
);
994 static int read_compacted_summaries(struct f2fs_sb_info
*sbi
)
996 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
997 struct curseg_info
*seg_i
;
998 unsigned char *kaddr
;
1003 start
= start_sum_block(sbi
);
1005 page
= get_meta_page(sbi
, start
++);
1006 kaddr
= (unsigned char *)page_address(page
);
1008 /* Step 1: restore nat cache */
1009 seg_i
= CURSEG_I(sbi
, CURSEG_HOT_DATA
);
1010 memcpy(&seg_i
->sum_blk
->n_nats
, kaddr
, SUM_JOURNAL_SIZE
);
1012 /* Step 2: restore sit cache */
1013 seg_i
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
1014 memcpy(&seg_i
->sum_blk
->n_sits
, kaddr
+ SUM_JOURNAL_SIZE
,
1016 offset
= 2 * SUM_JOURNAL_SIZE
;
1018 /* Step 3: restore summary entries */
1019 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
1020 unsigned short blk_off
;
1023 seg_i
= CURSEG_I(sbi
, i
);
1024 segno
= le32_to_cpu(ckpt
->cur_data_segno
[i
]);
1025 blk_off
= le16_to_cpu(ckpt
->cur_data_blkoff
[i
]);
1026 seg_i
->next_segno
= segno
;
1027 reset_curseg(sbi
, i
, 0);
1028 seg_i
->alloc_type
= ckpt
->alloc_type
[i
];
1029 seg_i
->next_blkoff
= blk_off
;
1031 if (seg_i
->alloc_type
== SSR
)
1032 blk_off
= sbi
->blocks_per_seg
;
1034 for (j
= 0; j
< blk_off
; j
++) {
1035 struct f2fs_summary
*s
;
1036 s
= (struct f2fs_summary
*)(kaddr
+ offset
);
1037 seg_i
->sum_blk
->entries
[j
] = *s
;
1038 offset
+= SUMMARY_SIZE
;
1039 if (offset
+ SUMMARY_SIZE
<= PAGE_CACHE_SIZE
-
1043 f2fs_put_page(page
, 1);
1046 page
= get_meta_page(sbi
, start
++);
1047 kaddr
= (unsigned char *)page_address(page
);
1051 f2fs_put_page(page
, 1);
1055 static int read_normal_summaries(struct f2fs_sb_info
*sbi
, int type
)
1057 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
1058 struct f2fs_summary_block
*sum
;
1059 struct curseg_info
*curseg
;
1061 unsigned short blk_off
;
1062 unsigned int segno
= 0;
1063 block_t blk_addr
= 0;
1065 /* get segment number and block addr */
1066 if (IS_DATASEG(type
)) {
1067 segno
= le32_to_cpu(ckpt
->cur_data_segno
[type
]);
1068 blk_off
= le16_to_cpu(ckpt
->cur_data_blkoff
[type
-
1070 if (is_set_ckpt_flags(ckpt
, CP_UMOUNT_FLAG
))
1071 blk_addr
= sum_blk_addr(sbi
, NR_CURSEG_TYPE
, type
);
1073 blk_addr
= sum_blk_addr(sbi
, NR_CURSEG_DATA_TYPE
, type
);
1075 segno
= le32_to_cpu(ckpt
->cur_node_segno
[type
-
1077 blk_off
= le16_to_cpu(ckpt
->cur_node_blkoff
[type
-
1079 if (is_set_ckpt_flags(ckpt
, CP_UMOUNT_FLAG
))
1080 blk_addr
= sum_blk_addr(sbi
, NR_CURSEG_NODE_TYPE
,
1081 type
- CURSEG_HOT_NODE
);
1083 blk_addr
= GET_SUM_BLOCK(sbi
, segno
);
1086 new = get_meta_page(sbi
, blk_addr
);
1087 sum
= (struct f2fs_summary_block
*)page_address(new);
1089 if (IS_NODESEG(type
)) {
1090 if (is_set_ckpt_flags(ckpt
, CP_UMOUNT_FLAG
)) {
1091 struct f2fs_summary
*ns
= &sum
->entries
[0];
1093 for (i
= 0; i
< sbi
->blocks_per_seg
; i
++, ns
++) {
1095 ns
->ofs_in_node
= 0;
1098 if (restore_node_summary(sbi
, segno
, sum
)) {
1099 f2fs_put_page(new, 1);
1105 /* set uncompleted segment to curseg */
1106 curseg
= CURSEG_I(sbi
, type
);
1107 mutex_lock(&curseg
->curseg_mutex
);
1108 memcpy(curseg
->sum_blk
, sum
, PAGE_CACHE_SIZE
);
1109 curseg
->next_segno
= segno
;
1110 reset_curseg(sbi
, type
, 0);
1111 curseg
->alloc_type
= ckpt
->alloc_type
[type
];
1112 curseg
->next_blkoff
= blk_off
;
1113 mutex_unlock(&curseg
->curseg_mutex
);
1114 f2fs_put_page(new, 1);
1118 static int restore_curseg_summaries(struct f2fs_sb_info
*sbi
)
1120 int type
= CURSEG_HOT_DATA
;
1122 if (is_set_ckpt_flags(F2FS_CKPT(sbi
), CP_COMPACT_SUM_FLAG
)) {
1123 /* restore for compacted data summary */
1124 if (read_compacted_summaries(sbi
))
1126 type
= CURSEG_HOT_NODE
;
1129 for (; type
<= CURSEG_COLD_NODE
; type
++)
1130 if (read_normal_summaries(sbi
, type
))
1135 static void write_compacted_summaries(struct f2fs_sb_info
*sbi
, block_t blkaddr
)
1138 unsigned char *kaddr
;
1139 struct f2fs_summary
*summary
;
1140 struct curseg_info
*seg_i
;
1141 int written_size
= 0;
1144 page
= grab_meta_page(sbi
, blkaddr
++);
1145 kaddr
= (unsigned char *)page_address(page
);
1147 /* Step 1: write nat cache */
1148 seg_i
= CURSEG_I(sbi
, CURSEG_HOT_DATA
);
1149 memcpy(kaddr
, &seg_i
->sum_blk
->n_nats
, SUM_JOURNAL_SIZE
);
1150 written_size
+= SUM_JOURNAL_SIZE
;
1152 /* Step 2: write sit cache */
1153 seg_i
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
1154 memcpy(kaddr
+ written_size
, &seg_i
->sum_blk
->n_sits
,
1156 written_size
+= SUM_JOURNAL_SIZE
;
1158 set_page_dirty(page
);
1160 /* Step 3: write summary entries */
1161 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
1162 unsigned short blkoff
;
1163 seg_i
= CURSEG_I(sbi
, i
);
1164 if (sbi
->ckpt
->alloc_type
[i
] == SSR
)
1165 blkoff
= sbi
->blocks_per_seg
;
1167 blkoff
= curseg_blkoff(sbi
, i
);
1169 for (j
= 0; j
< blkoff
; j
++) {
1171 page
= grab_meta_page(sbi
, blkaddr
++);
1172 kaddr
= (unsigned char *)page_address(page
);
1175 summary
= (struct f2fs_summary
*)(kaddr
+ written_size
);
1176 *summary
= seg_i
->sum_blk
->entries
[j
];
1177 written_size
+= SUMMARY_SIZE
;
1178 set_page_dirty(page
);
1180 if (written_size
+ SUMMARY_SIZE
<= PAGE_CACHE_SIZE
-
1184 f2fs_put_page(page
, 1);
1189 f2fs_put_page(page
, 1);
1192 static void write_normal_summaries(struct f2fs_sb_info
*sbi
,
1193 block_t blkaddr
, int type
)
1196 if (IS_DATASEG(type
))
1197 end
= type
+ NR_CURSEG_DATA_TYPE
;
1199 end
= type
+ NR_CURSEG_NODE_TYPE
;
1201 for (i
= type
; i
< end
; i
++) {
1202 struct curseg_info
*sum
= CURSEG_I(sbi
, i
);
1203 mutex_lock(&sum
->curseg_mutex
);
1204 write_sum_page(sbi
, sum
->sum_blk
, blkaddr
+ (i
- type
));
1205 mutex_unlock(&sum
->curseg_mutex
);
1209 void write_data_summaries(struct f2fs_sb_info
*sbi
, block_t start_blk
)
1211 if (is_set_ckpt_flags(F2FS_CKPT(sbi
), CP_COMPACT_SUM_FLAG
))
1212 write_compacted_summaries(sbi
, start_blk
);
1214 write_normal_summaries(sbi
, start_blk
, CURSEG_HOT_DATA
);
1217 void write_node_summaries(struct f2fs_sb_info
*sbi
, block_t start_blk
)
1219 if (is_set_ckpt_flags(F2FS_CKPT(sbi
), CP_UMOUNT_FLAG
))
1220 write_normal_summaries(sbi
, start_blk
, CURSEG_HOT_NODE
);
1224 int lookup_journal_in_cursum(struct f2fs_summary_block
*sum
, int type
,
1225 unsigned int val
, int alloc
)
1229 if (type
== NAT_JOURNAL
) {
1230 for (i
= 0; i
< nats_in_cursum(sum
); i
++) {
1231 if (le32_to_cpu(nid_in_journal(sum
, i
)) == val
)
1234 if (alloc
&& nats_in_cursum(sum
) < NAT_JOURNAL_ENTRIES
)
1235 return update_nats_in_cursum(sum
, 1);
1236 } else if (type
== SIT_JOURNAL
) {
1237 for (i
= 0; i
< sits_in_cursum(sum
); i
++)
1238 if (le32_to_cpu(segno_in_journal(sum
, i
)) == val
)
1240 if (alloc
&& sits_in_cursum(sum
) < SIT_JOURNAL_ENTRIES
)
1241 return update_sits_in_cursum(sum
, 1);
1246 static struct page
*get_current_sit_page(struct f2fs_sb_info
*sbi
,
1249 struct sit_info
*sit_i
= SIT_I(sbi
);
1250 unsigned int offset
= SIT_BLOCK_OFFSET(sit_i
, segno
);
1251 block_t blk_addr
= sit_i
->sit_base_addr
+ offset
;
1253 check_seg_range(sbi
, segno
);
1255 /* calculate sit block address */
1256 if (f2fs_test_bit(offset
, sit_i
->sit_bitmap
))
1257 blk_addr
+= sit_i
->sit_blocks
;
1259 return get_meta_page(sbi
, blk_addr
);
1262 static struct page
*get_next_sit_page(struct f2fs_sb_info
*sbi
,
1265 struct sit_info
*sit_i
= SIT_I(sbi
);
1266 struct page
*src_page
, *dst_page
;
1267 pgoff_t src_off
, dst_off
;
1268 void *src_addr
, *dst_addr
;
1270 src_off
= current_sit_addr(sbi
, start
);
1271 dst_off
= next_sit_addr(sbi
, src_off
);
1273 /* get current sit block page without lock */
1274 src_page
= get_meta_page(sbi
, src_off
);
1275 dst_page
= grab_meta_page(sbi
, dst_off
);
1276 BUG_ON(PageDirty(src_page
));
1278 src_addr
= page_address(src_page
);
1279 dst_addr
= page_address(dst_page
);
1280 memcpy(dst_addr
, src_addr
, PAGE_CACHE_SIZE
);
1282 set_page_dirty(dst_page
);
1283 f2fs_put_page(src_page
, 1);
1285 set_to_next_sit(sit_i
, start
);
1290 static bool flush_sits_in_journal(struct f2fs_sb_info
*sbi
)
1292 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
1293 struct f2fs_summary_block
*sum
= curseg
->sum_blk
;
1297 * If the journal area in the current summary is full of sit entries,
1298 * all the sit entries will be flushed. Otherwise the sit entries
1299 * are not able to replace with newly hot sit entries.
1301 if (sits_in_cursum(sum
) >= SIT_JOURNAL_ENTRIES
) {
1302 for (i
= sits_in_cursum(sum
) - 1; i
>= 0; i
--) {
1304 segno
= le32_to_cpu(segno_in_journal(sum
, i
));
1305 __mark_sit_entry_dirty(sbi
, segno
);
1307 update_sits_in_cursum(sum
, -sits_in_cursum(sum
));
1314 * CP calls this function, which flushes SIT entries including sit_journal,
1315 * and moves prefree segs to free segs.
1317 void flush_sit_entries(struct f2fs_sb_info
*sbi
)
1319 struct sit_info
*sit_i
= SIT_I(sbi
);
1320 unsigned long *bitmap
= sit_i
->dirty_sentries_bitmap
;
1321 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
1322 struct f2fs_summary_block
*sum
= curseg
->sum_blk
;
1323 unsigned long nsegs
= TOTAL_SEGS(sbi
);
1324 struct page
*page
= NULL
;
1325 struct f2fs_sit_block
*raw_sit
= NULL
;
1326 unsigned int start
= 0, end
= 0;
1327 unsigned int segno
= -1;
1330 mutex_lock(&curseg
->curseg_mutex
);
1331 mutex_lock(&sit_i
->sentry_lock
);
1334 * "flushed" indicates whether sit entries in journal are flushed
1335 * to the SIT area or not.
1337 flushed
= flush_sits_in_journal(sbi
);
1339 while ((segno
= find_next_bit(bitmap
, nsegs
, segno
+ 1)) < nsegs
) {
1340 struct seg_entry
*se
= get_seg_entry(sbi
, segno
);
1341 int sit_offset
, offset
;
1343 sit_offset
= SIT_ENTRY_OFFSET(sit_i
, segno
);
1348 offset
= lookup_journal_in_cursum(sum
, SIT_JOURNAL
, segno
, 1);
1350 segno_in_journal(sum
, offset
) = cpu_to_le32(segno
);
1351 seg_info_to_raw_sit(se
, &sit_in_journal(sum
, offset
));
1355 if (!page
|| (start
> segno
) || (segno
> end
)) {
1357 f2fs_put_page(page
, 1);
1361 start
= START_SEGNO(sit_i
, segno
);
1362 end
= start
+ SIT_ENTRY_PER_BLOCK
- 1;
1364 /* read sit block that will be updated */
1365 page
= get_next_sit_page(sbi
, start
);
1366 raw_sit
= page_address(page
);
1369 /* udpate entry in SIT block */
1370 seg_info_to_raw_sit(se
, &raw_sit
->entries
[sit_offset
]);
1372 __clear_bit(segno
, bitmap
);
1373 sit_i
->dirty_sentries
--;
1375 mutex_unlock(&sit_i
->sentry_lock
);
1376 mutex_unlock(&curseg
->curseg_mutex
);
1378 /* writeout last modified SIT block */
1379 f2fs_put_page(page
, 1);
1381 set_prefree_as_free_segments(sbi
);
1384 static int build_sit_info(struct f2fs_sb_info
*sbi
)
1386 struct f2fs_super_block
*raw_super
= F2FS_RAW_SUPER(sbi
);
1387 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
1388 struct sit_info
*sit_i
;
1389 unsigned int sit_segs
, start
;
1390 char *src_bitmap
, *dst_bitmap
;
1391 unsigned int bitmap_size
;
1393 /* allocate memory for SIT information */
1394 sit_i
= kzalloc(sizeof(struct sit_info
), GFP_KERNEL
);
1398 SM_I(sbi
)->sit_info
= sit_i
;
1400 sit_i
->sentries
= vzalloc(TOTAL_SEGS(sbi
) * sizeof(struct seg_entry
));
1401 if (!sit_i
->sentries
)
1404 bitmap_size
= f2fs_bitmap_size(TOTAL_SEGS(sbi
));
1405 sit_i
->dirty_sentries_bitmap
= kzalloc(bitmap_size
, GFP_KERNEL
);
1406 if (!sit_i
->dirty_sentries_bitmap
)
1409 for (start
= 0; start
< TOTAL_SEGS(sbi
); start
++) {
1410 sit_i
->sentries
[start
].cur_valid_map
1411 = kzalloc(SIT_VBLOCK_MAP_SIZE
, GFP_KERNEL
);
1412 sit_i
->sentries
[start
].ckpt_valid_map
1413 = kzalloc(SIT_VBLOCK_MAP_SIZE
, GFP_KERNEL
);
1414 if (!sit_i
->sentries
[start
].cur_valid_map
1415 || !sit_i
->sentries
[start
].ckpt_valid_map
)
1419 if (sbi
->segs_per_sec
> 1) {
1420 sit_i
->sec_entries
= vzalloc(sbi
->total_sections
*
1421 sizeof(struct sec_entry
));
1422 if (!sit_i
->sec_entries
)
1426 /* get information related with SIT */
1427 sit_segs
= le32_to_cpu(raw_super
->segment_count_sit
) >> 1;
1429 /* setup SIT bitmap from ckeckpoint pack */
1430 bitmap_size
= __bitmap_size(sbi
, SIT_BITMAP
);
1431 src_bitmap
= __bitmap_ptr(sbi
, SIT_BITMAP
);
1433 dst_bitmap
= kzalloc(bitmap_size
, GFP_KERNEL
);
1436 memcpy(dst_bitmap
, src_bitmap
, bitmap_size
);
1438 /* init SIT information */
1439 sit_i
->s_ops
= &default_salloc_ops
;
1441 sit_i
->sit_base_addr
= le32_to_cpu(raw_super
->sit_blkaddr
);
1442 sit_i
->sit_blocks
= sit_segs
<< sbi
->log_blocks_per_seg
;
1443 sit_i
->written_valid_blocks
= le64_to_cpu(ckpt
->valid_block_count
);
1444 sit_i
->sit_bitmap
= dst_bitmap
;
1445 sit_i
->bitmap_size
= bitmap_size
;
1446 sit_i
->dirty_sentries
= 0;
1447 sit_i
->sents_per_block
= SIT_ENTRY_PER_BLOCK
;
1448 sit_i
->elapsed_time
= le64_to_cpu(sbi
->ckpt
->elapsed_time
);
1449 sit_i
->mounted_time
= CURRENT_TIME_SEC
.tv_sec
;
1450 mutex_init(&sit_i
->sentry_lock
);
1454 static int build_free_segmap(struct f2fs_sb_info
*sbi
)
1456 struct f2fs_sm_info
*sm_info
= SM_I(sbi
);
1457 struct free_segmap_info
*free_i
;
1458 unsigned int bitmap_size
, sec_bitmap_size
;
1460 /* allocate memory for free segmap information */
1461 free_i
= kzalloc(sizeof(struct free_segmap_info
), GFP_KERNEL
);
1465 SM_I(sbi
)->free_info
= free_i
;
1467 bitmap_size
= f2fs_bitmap_size(TOTAL_SEGS(sbi
));
1468 free_i
->free_segmap
= kmalloc(bitmap_size
, GFP_KERNEL
);
1469 if (!free_i
->free_segmap
)
1472 sec_bitmap_size
= f2fs_bitmap_size(sbi
->total_sections
);
1473 free_i
->free_secmap
= kmalloc(sec_bitmap_size
, GFP_KERNEL
);
1474 if (!free_i
->free_secmap
)
1477 /* set all segments as dirty temporarily */
1478 memset(free_i
->free_segmap
, 0xff, bitmap_size
);
1479 memset(free_i
->free_secmap
, 0xff, sec_bitmap_size
);
1481 /* init free segmap information */
1482 free_i
->start_segno
=
1483 (unsigned int) GET_SEGNO_FROM_SEG0(sbi
, sm_info
->main_blkaddr
);
1484 free_i
->free_segments
= 0;
1485 free_i
->free_sections
= 0;
1486 rwlock_init(&free_i
->segmap_lock
);
1490 static int build_curseg(struct f2fs_sb_info
*sbi
)
1492 struct curseg_info
*array
= NULL
;
1495 array
= kzalloc(sizeof(*array
) * NR_CURSEG_TYPE
, GFP_KERNEL
);
1499 SM_I(sbi
)->curseg_array
= array
;
1501 for (i
= 0; i
< NR_CURSEG_TYPE
; i
++) {
1502 mutex_init(&array
[i
].curseg_mutex
);
1503 array
[i
].sum_blk
= kzalloc(PAGE_CACHE_SIZE
, GFP_KERNEL
);
1504 if (!array
[i
].sum_blk
)
1506 array
[i
].segno
= NULL_SEGNO
;
1507 array
[i
].next_blkoff
= 0;
1509 return restore_curseg_summaries(sbi
);
1512 static void build_sit_entries(struct f2fs_sb_info
*sbi
)
1514 struct sit_info
*sit_i
= SIT_I(sbi
);
1515 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
1516 struct f2fs_summary_block
*sum
= curseg
->sum_blk
;
1519 for (start
= 0; start
< TOTAL_SEGS(sbi
); start
++) {
1520 struct seg_entry
*se
= &sit_i
->sentries
[start
];
1521 struct f2fs_sit_block
*sit_blk
;
1522 struct f2fs_sit_entry sit
;
1526 mutex_lock(&curseg
->curseg_mutex
);
1527 for (i
= 0; i
< sits_in_cursum(sum
); i
++) {
1528 if (le32_to_cpu(segno_in_journal(sum
, i
)) == start
) {
1529 sit
= sit_in_journal(sum
, i
);
1530 mutex_unlock(&curseg
->curseg_mutex
);
1534 mutex_unlock(&curseg
->curseg_mutex
);
1535 page
= get_current_sit_page(sbi
, start
);
1536 sit_blk
= (struct f2fs_sit_block
*)page_address(page
);
1537 sit
= sit_blk
->entries
[SIT_ENTRY_OFFSET(sit_i
, start
)];
1538 f2fs_put_page(page
, 1);
1540 check_block_count(sbi
, start
, &sit
);
1541 seg_info_from_raw_sit(se
, &sit
);
1542 if (sbi
->segs_per_sec
> 1) {
1543 struct sec_entry
*e
= get_sec_entry(sbi
, start
);
1544 e
->valid_blocks
+= se
->valid_blocks
;
1549 static void init_free_segmap(struct f2fs_sb_info
*sbi
)
1554 for (start
= 0; start
< TOTAL_SEGS(sbi
); start
++) {
1555 struct seg_entry
*sentry
= get_seg_entry(sbi
, start
);
1556 if (!sentry
->valid_blocks
)
1557 __set_free(sbi
, start
);
1560 /* set use the current segments */
1561 for (type
= CURSEG_HOT_DATA
; type
<= CURSEG_COLD_NODE
; type
++) {
1562 struct curseg_info
*curseg_t
= CURSEG_I(sbi
, type
);
1563 __set_test_and_inuse(sbi
, curseg_t
->segno
);
1567 static void init_dirty_segmap(struct f2fs_sb_info
*sbi
)
1569 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
1570 struct free_segmap_info
*free_i
= FREE_I(sbi
);
1571 unsigned int segno
= 0, offset
= 0;
1572 unsigned short valid_blocks
;
1574 while (segno
< TOTAL_SEGS(sbi
)) {
1575 /* find dirty segment based on free segmap */
1576 segno
= find_next_inuse(free_i
, TOTAL_SEGS(sbi
), offset
);
1577 if (segno
>= TOTAL_SEGS(sbi
))
1580 valid_blocks
= get_valid_blocks(sbi
, segno
, 0);
1581 if (valid_blocks
>= sbi
->blocks_per_seg
|| !valid_blocks
)
1583 mutex_lock(&dirty_i
->seglist_lock
);
1584 __locate_dirty_segment(sbi
, segno
, DIRTY
);
1585 mutex_unlock(&dirty_i
->seglist_lock
);
1589 static int init_victim_segmap(struct f2fs_sb_info
*sbi
)
1591 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
1592 unsigned int bitmap_size
= f2fs_bitmap_size(TOTAL_SEGS(sbi
));
1594 dirty_i
->victim_segmap
[FG_GC
] = kzalloc(bitmap_size
, GFP_KERNEL
);
1595 dirty_i
->victim_segmap
[BG_GC
] = kzalloc(bitmap_size
, GFP_KERNEL
);
1596 if (!dirty_i
->victim_segmap
[FG_GC
] || !dirty_i
->victim_segmap
[BG_GC
])
1601 static int build_dirty_segmap(struct f2fs_sb_info
*sbi
)
1603 struct dirty_seglist_info
*dirty_i
;
1604 unsigned int bitmap_size
, i
;
1606 /* allocate memory for dirty segments list information */
1607 dirty_i
= kzalloc(sizeof(struct dirty_seglist_info
), GFP_KERNEL
);
1611 SM_I(sbi
)->dirty_info
= dirty_i
;
1612 mutex_init(&dirty_i
->seglist_lock
);
1614 bitmap_size
= f2fs_bitmap_size(TOTAL_SEGS(sbi
));
1616 for (i
= 0; i
< NR_DIRTY_TYPE
; i
++) {
1617 dirty_i
->dirty_segmap
[i
] = kzalloc(bitmap_size
, GFP_KERNEL
);
1618 dirty_i
->nr_dirty
[i
] = 0;
1619 if (!dirty_i
->dirty_segmap
[i
])
1623 init_dirty_segmap(sbi
);
1624 return init_victim_segmap(sbi
);
1628 * Update min, max modified time for cost-benefit GC algorithm
1630 static void init_min_max_mtime(struct f2fs_sb_info
*sbi
)
1632 struct sit_info
*sit_i
= SIT_I(sbi
);
1635 mutex_lock(&sit_i
->sentry_lock
);
1637 sit_i
->min_mtime
= LLONG_MAX
;
1639 for (segno
= 0; segno
< TOTAL_SEGS(sbi
); segno
+= sbi
->segs_per_sec
) {
1641 unsigned long long mtime
= 0;
1643 for (i
= 0; i
< sbi
->segs_per_sec
; i
++)
1644 mtime
+= get_seg_entry(sbi
, segno
+ i
)->mtime
;
1646 mtime
= div_u64(mtime
, sbi
->segs_per_sec
);
1648 if (sit_i
->min_mtime
> mtime
)
1649 sit_i
->min_mtime
= mtime
;
1651 sit_i
->max_mtime
= get_mtime(sbi
);
1652 mutex_unlock(&sit_i
->sentry_lock
);
1655 int build_segment_manager(struct f2fs_sb_info
*sbi
)
1657 struct f2fs_super_block
*raw_super
= F2FS_RAW_SUPER(sbi
);
1658 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
1659 struct f2fs_sm_info
*sm_info
= NULL
;
1662 sm_info
= kzalloc(sizeof(struct f2fs_sm_info
), GFP_KERNEL
);
1667 sbi
->sm_info
= sm_info
;
1668 INIT_LIST_HEAD(&sm_info
->wblist_head
);
1669 spin_lock_init(&sm_info
->wblist_lock
);
1670 sm_info
->seg0_blkaddr
= le32_to_cpu(raw_super
->segment0_blkaddr
);
1671 sm_info
->main_blkaddr
= le32_to_cpu(raw_super
->main_blkaddr
);
1672 sm_info
->segment_count
= le32_to_cpu(raw_super
->segment_count
);
1673 sm_info
->reserved_segments
= le32_to_cpu(ckpt
->rsvd_segment_count
);
1674 sm_info
->ovp_segments
= le32_to_cpu(ckpt
->overprov_segment_count
);
1675 sm_info
->main_segments
= le32_to_cpu(raw_super
->segment_count_main
);
1676 sm_info
->ssa_blkaddr
= le32_to_cpu(raw_super
->ssa_blkaddr
);
1678 err
= build_sit_info(sbi
);
1681 err
= build_free_segmap(sbi
);
1684 err
= build_curseg(sbi
);
1688 /* reinit free segmap based on SIT */
1689 build_sit_entries(sbi
);
1691 init_free_segmap(sbi
);
1692 err
= build_dirty_segmap(sbi
);
1696 init_min_max_mtime(sbi
);
1700 static void discard_dirty_segmap(struct f2fs_sb_info
*sbi
,
1701 enum dirty_type dirty_type
)
1703 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
1705 mutex_lock(&dirty_i
->seglist_lock
);
1706 kfree(dirty_i
->dirty_segmap
[dirty_type
]);
1707 dirty_i
->nr_dirty
[dirty_type
] = 0;
1708 mutex_unlock(&dirty_i
->seglist_lock
);
1711 void reset_victim_segmap(struct f2fs_sb_info
*sbi
)
1713 unsigned int bitmap_size
= f2fs_bitmap_size(TOTAL_SEGS(sbi
));
1714 memset(DIRTY_I(sbi
)->victim_segmap
[FG_GC
], 0, bitmap_size
);
1717 static void destroy_victim_segmap(struct f2fs_sb_info
*sbi
)
1719 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
1721 kfree(dirty_i
->victim_segmap
[FG_GC
]);
1722 kfree(dirty_i
->victim_segmap
[BG_GC
]);
1725 static void destroy_dirty_segmap(struct f2fs_sb_info
*sbi
)
1727 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
1733 /* discard pre-free/dirty segments list */
1734 for (i
= 0; i
< NR_DIRTY_TYPE
; i
++)
1735 discard_dirty_segmap(sbi
, i
);
1737 destroy_victim_segmap(sbi
);
1738 SM_I(sbi
)->dirty_info
= NULL
;
1742 static void destroy_curseg(struct f2fs_sb_info
*sbi
)
1744 struct curseg_info
*array
= SM_I(sbi
)->curseg_array
;
1749 SM_I(sbi
)->curseg_array
= NULL
;
1750 for (i
= 0; i
< NR_CURSEG_TYPE
; i
++)
1751 kfree(array
[i
].sum_blk
);
1755 static void destroy_free_segmap(struct f2fs_sb_info
*sbi
)
1757 struct free_segmap_info
*free_i
= SM_I(sbi
)->free_info
;
1760 SM_I(sbi
)->free_info
= NULL
;
1761 kfree(free_i
->free_segmap
);
1762 kfree(free_i
->free_secmap
);
1766 static void destroy_sit_info(struct f2fs_sb_info
*sbi
)
1768 struct sit_info
*sit_i
= SIT_I(sbi
);
1774 if (sit_i
->sentries
) {
1775 for (start
= 0; start
< TOTAL_SEGS(sbi
); start
++) {
1776 kfree(sit_i
->sentries
[start
].cur_valid_map
);
1777 kfree(sit_i
->sentries
[start
].ckpt_valid_map
);
1780 vfree(sit_i
->sentries
);
1781 vfree(sit_i
->sec_entries
);
1782 kfree(sit_i
->dirty_sentries_bitmap
);
1784 SM_I(sbi
)->sit_info
= NULL
;
1785 kfree(sit_i
->sit_bitmap
);
1789 void destroy_segment_manager(struct f2fs_sb_info
*sbi
)
1791 struct f2fs_sm_info
*sm_info
= SM_I(sbi
);
1792 destroy_dirty_segmap(sbi
);
1793 destroy_curseg(sbi
);
1794 destroy_free_segmap(sbi
);
1795 destroy_sit_info(sbi
);
1796 sbi
->sm_info
= NULL
;