4 * Copyright (C) 2002, Linus Torvalds.
6 * Contains all the functions related to writing back and waiting
7 * upon dirty inodes against superblocks, and writing back dirty
8 * pages against inodes. ie: data writeback. Writeout of the
9 * inode itself is not handled here.
11 * 10Apr2002 Andrew Morton
12 * Split out of fs/inode.c
13 * Additions for address_space-based writeback
16 #include <linux/kernel.h>
17 #include <linux/export.h>
18 #include <linux/spinlock.h>
19 #include <linux/slab.h>
20 #include <linux/sched.h>
23 #include <linux/pagemap.h>
24 #include <linux/kthread.h>
25 #include <linux/freezer.h>
26 #include <linux/writeback.h>
27 #include <linux/blkdev.h>
28 #include <linux/backing-dev.h>
29 #include <linux/tracepoint.h>
33 * 4MB minimal write chunk size
35 #define MIN_WRITEBACK_PAGES (4096UL >> (PAGE_CACHE_SHIFT - 10))
38 * Passed into wb_writeback(), essentially a subset of writeback_control
40 struct wb_writeback_work
{
42 struct super_block
*sb
;
43 unsigned long *older_than_this
;
44 enum writeback_sync_modes sync_mode
;
45 unsigned int tagged_writepages
:1;
46 unsigned int for_kupdate
:1;
47 unsigned int range_cyclic
:1;
48 unsigned int for_background
:1;
49 enum wb_reason reason
; /* why was writeback initiated? */
51 struct list_head list
; /* pending work list */
52 struct completion
*done
; /* set if the caller waits */
56 * We don't actually have pdflush, but this one is exported though /proc...
58 int nr_pdflush_threads
;
61 * writeback_in_progress - determine whether there is writeback in progress
62 * @bdi: the device's backing_dev_info structure.
64 * Determine whether there is writeback waiting to be handled against a
67 int writeback_in_progress(struct backing_dev_info
*bdi
)
69 return test_bit(BDI_writeback_running
, &bdi
->state
);
72 static inline struct backing_dev_info
*inode_to_bdi(struct inode
*inode
)
74 struct super_block
*sb
= inode
->i_sb
;
76 if (strcmp(sb
->s_type
->name
, "bdev") == 0)
77 return inode
->i_mapping
->backing_dev_info
;
82 static inline struct inode
*wb_inode(struct list_head
*head
)
84 return list_entry(head
, struct inode
, i_wb_list
);
88 * Include the creation of the trace points after defining the
89 * wb_writeback_work structure and inline functions so that the definition
90 * remains local to this file.
92 #define CREATE_TRACE_POINTS
93 #include <trace/events/writeback.h>
95 /* Wakeup flusher thread or forker thread to fork it. Requires bdi->wb_lock. */
96 static void bdi_wakeup_flusher(struct backing_dev_info
*bdi
)
99 wake_up_process(bdi
->wb
.task
);
102 * The bdi thread isn't there, wake up the forker thread which
103 * will create and run it.
105 wake_up_process(default_backing_dev_info
.wb
.task
);
109 static void bdi_queue_work(struct backing_dev_info
*bdi
,
110 struct wb_writeback_work
*work
)
112 trace_writeback_queue(bdi
, work
);
114 spin_lock_bh(&bdi
->wb_lock
);
115 list_add_tail(&work
->list
, &bdi
->work_list
);
117 trace_writeback_nothread(bdi
, work
);
118 bdi_wakeup_flusher(bdi
);
119 spin_unlock_bh(&bdi
->wb_lock
);
123 __bdi_start_writeback(struct backing_dev_info
*bdi
, long nr_pages
,
124 bool range_cyclic
, enum wb_reason reason
)
126 struct wb_writeback_work
*work
;
129 * This is WB_SYNC_NONE writeback, so if allocation fails just
130 * wakeup the thread for old dirty data writeback
132 work
= kzalloc(sizeof(*work
), GFP_ATOMIC
);
135 trace_writeback_nowork(bdi
);
136 wake_up_process(bdi
->wb
.task
);
141 work
->sync_mode
= WB_SYNC_NONE
;
142 work
->nr_pages
= nr_pages
;
143 work
->range_cyclic
= range_cyclic
;
144 work
->reason
= reason
;
146 bdi_queue_work(bdi
, work
);
150 * bdi_start_writeback - start writeback
151 * @bdi: the backing device to write from
152 * @nr_pages: the number of pages to write
153 * @reason: reason why some writeback work was initiated
156 * This does WB_SYNC_NONE opportunistic writeback. The IO is only
157 * started when this function returns, we make no guarantees on
158 * completion. Caller need not hold sb s_umount semaphore.
161 void bdi_start_writeback(struct backing_dev_info
*bdi
, long nr_pages
,
162 enum wb_reason reason
)
164 __bdi_start_writeback(bdi
, nr_pages
, true, reason
);
168 * bdi_start_background_writeback - start background writeback
169 * @bdi: the backing device to write from
172 * This makes sure WB_SYNC_NONE background writeback happens. When
173 * this function returns, it is only guaranteed that for given BDI
174 * some IO is happening if we are over background dirty threshold.
175 * Caller need not hold sb s_umount semaphore.
177 void bdi_start_background_writeback(struct backing_dev_info
*bdi
)
180 * We just wake up the flusher thread. It will perform background
181 * writeback as soon as there is no other work to do.
183 trace_writeback_wake_background(bdi
);
184 spin_lock_bh(&bdi
->wb_lock
);
185 bdi_wakeup_flusher(bdi
);
186 spin_unlock_bh(&bdi
->wb_lock
);
190 * Remove the inode from the writeback list it is on.
192 void inode_wb_list_del(struct inode
*inode
)
194 struct backing_dev_info
*bdi
= inode_to_bdi(inode
);
196 spin_lock(&bdi
->wb
.list_lock
);
197 list_del_init(&inode
->i_wb_list
);
198 spin_unlock(&bdi
->wb
.list_lock
);
202 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
203 * furthest end of its superblock's dirty-inode list.
205 * Before stamping the inode's ->dirtied_when, we check to see whether it is
206 * already the most-recently-dirtied inode on the b_dirty list. If that is
207 * the case then the inode must have been redirtied while it was being written
208 * out and we don't reset its dirtied_when.
210 static void redirty_tail(struct inode
*inode
, struct bdi_writeback
*wb
)
212 assert_spin_locked(&wb
->list_lock
);
213 if (!list_empty(&wb
->b_dirty
)) {
216 tail
= wb_inode(wb
->b_dirty
.next
);
217 if (time_before(inode
->dirtied_when
, tail
->dirtied_when
))
218 inode
->dirtied_when
= jiffies
;
220 list_move(&inode
->i_wb_list
, &wb
->b_dirty
);
224 * requeue inode for re-scanning after bdi->b_io list is exhausted.
226 static void requeue_io(struct inode
*inode
, struct bdi_writeback
*wb
)
228 assert_spin_locked(&wb
->list_lock
);
229 list_move(&inode
->i_wb_list
, &wb
->b_more_io
);
232 static void inode_sync_complete(struct inode
*inode
)
234 inode
->i_state
&= ~I_SYNC
;
235 /* Waiters must see I_SYNC cleared before being woken up */
237 wake_up_bit(&inode
->i_state
, __I_SYNC
);
240 static bool inode_dirtied_after(struct inode
*inode
, unsigned long t
)
242 bool ret
= time_after(inode
->dirtied_when
, t
);
245 * For inodes being constantly redirtied, dirtied_when can get stuck.
246 * It _appears_ to be in the future, but is actually in distant past.
247 * This test is necessary to prevent such wrapped-around relative times
248 * from permanently stopping the whole bdi writeback.
250 ret
= ret
&& time_before_eq(inode
->dirtied_when
, jiffies
);
256 * Move expired (dirtied after work->older_than_this) dirty inodes from
257 * @delaying_queue to @dispatch_queue.
259 static int move_expired_inodes(struct list_head
*delaying_queue
,
260 struct list_head
*dispatch_queue
,
261 struct wb_writeback_work
*work
)
264 struct list_head
*pos
, *node
;
265 struct super_block
*sb
= NULL
;
270 while (!list_empty(delaying_queue
)) {
271 inode
= wb_inode(delaying_queue
->prev
);
272 if (work
->older_than_this
&&
273 inode_dirtied_after(inode
, *work
->older_than_this
))
275 if (sb
&& sb
!= inode
->i_sb
)
278 list_move(&inode
->i_wb_list
, &tmp
);
282 /* just one sb in list, splice to dispatch_queue and we're done */
284 list_splice(&tmp
, dispatch_queue
);
288 /* Move inodes from one superblock together */
289 while (!list_empty(&tmp
)) {
290 sb
= wb_inode(tmp
.prev
)->i_sb
;
291 list_for_each_prev_safe(pos
, node
, &tmp
) {
292 inode
= wb_inode(pos
);
293 if (inode
->i_sb
== sb
)
294 list_move(&inode
->i_wb_list
, dispatch_queue
);
302 * Queue all expired dirty inodes for io, eldest first.
304 * newly dirtied b_dirty b_io b_more_io
305 * =============> gf edc BA
307 * newly dirtied b_dirty b_io b_more_io
308 * =============> g fBAedc
310 * +--> dequeue for IO
312 static void queue_io(struct bdi_writeback
*wb
, struct wb_writeback_work
*work
)
315 assert_spin_locked(&wb
->list_lock
);
316 list_splice_init(&wb
->b_more_io
, &wb
->b_io
);
317 moved
= move_expired_inodes(&wb
->b_dirty
, &wb
->b_io
, work
);
318 trace_writeback_queue_io(wb
, work
, moved
);
321 static int write_inode(struct inode
*inode
, struct writeback_control
*wbc
)
323 if (inode
->i_sb
->s_op
->write_inode
&& !is_bad_inode(inode
))
324 return inode
->i_sb
->s_op
->write_inode(inode
, wbc
);
329 * Wait for writeback on an inode to complete.
331 static void inode_wait_for_writeback(struct inode
*inode
,
332 struct bdi_writeback
*wb
)
334 DEFINE_WAIT_BIT(wq
, &inode
->i_state
, __I_SYNC
);
335 wait_queue_head_t
*wqh
;
337 wqh
= bit_waitqueue(&inode
->i_state
, __I_SYNC
);
338 while (inode
->i_state
& I_SYNC
) {
339 spin_unlock(&inode
->i_lock
);
340 spin_unlock(&wb
->list_lock
);
341 __wait_on_bit(wqh
, &wq
, inode_wait
, TASK_UNINTERRUPTIBLE
);
342 spin_lock(&wb
->list_lock
);
343 spin_lock(&inode
->i_lock
);
348 * Write out an inode's dirty pages. Called under wb->list_lock and
349 * inode->i_lock. Either the caller has an active reference on the inode or
350 * the inode has I_WILL_FREE set.
352 * If `wait' is set, wait on the writeout.
354 * The whole writeout design is quite complex and fragile. We want to avoid
355 * starvation of particular inodes when others are being redirtied, prevent
359 writeback_single_inode(struct inode
*inode
, struct bdi_writeback
*wb
,
360 struct writeback_control
*wbc
)
362 struct address_space
*mapping
= inode
->i_mapping
;
363 long nr_to_write
= wbc
->nr_to_write
;
367 assert_spin_locked(&wb
->list_lock
);
368 assert_spin_locked(&inode
->i_lock
);
370 if (!atomic_read(&inode
->i_count
))
371 WARN_ON(!(inode
->i_state
& (I_WILL_FREE
|I_FREEING
)));
373 WARN_ON(inode
->i_state
& I_WILL_FREE
);
375 if (inode
->i_state
& I_SYNC
) {
376 if (wbc
->sync_mode
!= WB_SYNC_ALL
)
379 * It's a data-integrity sync. We must wait.
381 inode_wait_for_writeback(inode
, wb
);
384 BUG_ON(inode
->i_state
& I_SYNC
);
386 /* Set I_SYNC, reset I_DIRTY_PAGES */
387 inode
->i_state
|= I_SYNC
;
388 inode
->i_state
&= ~I_DIRTY_PAGES
;
389 spin_unlock(&inode
->i_lock
);
390 spin_unlock(&wb
->list_lock
);
392 ret
= do_writepages(mapping
, wbc
);
395 * Make sure to wait on the data before writing out the metadata.
396 * This is important for filesystems that modify metadata on data
399 if (wbc
->sync_mode
== WB_SYNC_ALL
) {
400 int err
= filemap_fdatawait(mapping
);
406 * Some filesystems may redirty the inode during the writeback
407 * due to delalloc, clear dirty metadata flags right before
410 spin_lock(&inode
->i_lock
);
411 dirty
= inode
->i_state
& I_DIRTY
;
412 inode
->i_state
&= ~(I_DIRTY_SYNC
| I_DIRTY_DATASYNC
);
413 spin_unlock(&inode
->i_lock
);
414 /* Don't write the inode if only I_DIRTY_PAGES was set */
415 if (dirty
& (I_DIRTY_SYNC
| I_DIRTY_DATASYNC
)) {
416 int err
= write_inode(inode
, wbc
);
421 spin_lock(&wb
->list_lock
);
422 spin_lock(&inode
->i_lock
);
423 if (!(inode
->i_state
& I_FREEING
)) {
425 * Sync livelock prevention. Each inode is tagged and synced in
426 * one shot. If still dirty, it will be redirty_tail()'ed below.
427 * Update the dirty time to prevent enqueue and sync it again.
429 if ((inode
->i_state
& I_DIRTY
) &&
430 (wbc
->sync_mode
== WB_SYNC_ALL
|| wbc
->tagged_writepages
))
431 inode
->dirtied_when
= jiffies
;
433 if (mapping_tagged(mapping
, PAGECACHE_TAG_DIRTY
)) {
435 * We didn't write back all the pages. nfs_writepages()
436 * sometimes bales out without doing anything.
438 inode
->i_state
|= I_DIRTY_PAGES
;
439 if (wbc
->nr_to_write
<= 0) {
441 * slice used up: queue for next turn
443 requeue_io(inode
, wb
);
446 * Writeback blocked by something other than
447 * congestion. Delay the inode for some time to
448 * avoid spinning on the CPU (100% iowait)
449 * retrying writeback of the dirty page/inode
450 * that cannot be performed immediately.
452 redirty_tail(inode
, wb
);
454 } else if (inode
->i_state
& I_DIRTY
) {
456 * Filesystems can dirty the inode during writeback
457 * operations, such as delayed allocation during
458 * submission or metadata updates after data IO
461 redirty_tail(inode
, wb
);
464 * The inode is clean. At this point we either have
465 * a reference to the inode or it's on it's way out.
466 * No need to add it back to the LRU.
468 list_del_init(&inode
->i_wb_list
);
471 inode_sync_complete(inode
);
472 trace_writeback_single_inode(inode
, wbc
, nr_to_write
);
476 static long writeback_chunk_size(struct backing_dev_info
*bdi
,
477 struct wb_writeback_work
*work
)
482 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
483 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
484 * here avoids calling into writeback_inodes_wb() more than once.
486 * The intended call sequence for WB_SYNC_ALL writeback is:
489 * writeback_sb_inodes() <== called only once
490 * write_cache_pages() <== called once for each inode
491 * (quickly) tag currently dirty pages
492 * (maybe slowly) sync all tagged pages
494 if (work
->sync_mode
== WB_SYNC_ALL
|| work
->tagged_writepages
)
497 pages
= min(bdi
->avg_write_bandwidth
/ 2,
498 global_dirty_limit
/ DIRTY_SCOPE
);
499 pages
= min(pages
, work
->nr_pages
);
500 pages
= round_down(pages
+ MIN_WRITEBACK_PAGES
,
501 MIN_WRITEBACK_PAGES
);
508 * Write a portion of b_io inodes which belong to @sb.
510 * If @only_this_sb is true, then find and write all such
511 * inodes. Otherwise write only ones which go sequentially
514 * Return the number of pages and/or inodes written.
516 static long writeback_sb_inodes(struct super_block
*sb
,
517 struct bdi_writeback
*wb
,
518 struct wb_writeback_work
*work
)
520 struct writeback_control wbc
= {
521 .sync_mode
= work
->sync_mode
,
522 .tagged_writepages
= work
->tagged_writepages
,
523 .for_kupdate
= work
->for_kupdate
,
524 .for_background
= work
->for_background
,
525 .range_cyclic
= work
->range_cyclic
,
527 .range_end
= LLONG_MAX
,
529 unsigned long start_time
= jiffies
;
531 long wrote
= 0; /* count both pages and inodes */
533 while (!list_empty(&wb
->b_io
)) {
534 struct inode
*inode
= wb_inode(wb
->b_io
.prev
);
536 if (inode
->i_sb
!= sb
) {
539 * We only want to write back data for this
540 * superblock, move all inodes not belonging
541 * to it back onto the dirty list.
543 redirty_tail(inode
, wb
);
548 * The inode belongs to a different superblock.
549 * Bounce back to the caller to unpin this and
550 * pin the next superblock.
556 * Don't bother with new inodes or inodes beeing freed, first
557 * kind does not need peridic writeout yet, and for the latter
558 * kind writeout is handled by the freer.
560 spin_lock(&inode
->i_lock
);
561 if (inode
->i_state
& (I_NEW
| I_FREEING
| I_WILL_FREE
)) {
562 spin_unlock(&inode
->i_lock
);
563 redirty_tail(inode
, wb
);
566 if ((inode
->i_state
& I_SYNC
) && wbc
.sync_mode
!= WB_SYNC_ALL
) {
568 * If this inode is locked for writeback and we are not
569 * doing writeback-for-data-integrity, move it to
570 * b_more_io so that writeback can proceed with the
571 * other inodes on s_io.
573 * We'll have another go at writing back this inode
574 * when we completed a full scan of b_io.
576 spin_unlock(&inode
->i_lock
);
577 requeue_io(inode
, wb
);
578 trace_writeback_sb_inodes_requeue(inode
);
582 write_chunk
= writeback_chunk_size(wb
->bdi
, work
);
583 wbc
.nr_to_write
= write_chunk
;
584 wbc
.pages_skipped
= 0;
586 writeback_single_inode(inode
, wb
, &wbc
);
588 work
->nr_pages
-= write_chunk
- wbc
.nr_to_write
;
589 wrote
+= write_chunk
- wbc
.nr_to_write
;
590 if (!(inode
->i_state
& I_DIRTY
))
592 if (wbc
.pages_skipped
) {
594 * writeback is not making progress due to locked
595 * buffers. Skip this inode for now.
597 redirty_tail(inode
, wb
);
599 spin_unlock(&inode
->i_lock
);
600 spin_unlock(&wb
->list_lock
);
603 spin_lock(&wb
->list_lock
);
605 * bail out to wb_writeback() often enough to check
606 * background threshold and other termination conditions.
609 if (time_is_before_jiffies(start_time
+ HZ
/ 10UL))
611 if (work
->nr_pages
<= 0)
618 static long __writeback_inodes_wb(struct bdi_writeback
*wb
,
619 struct wb_writeback_work
*work
)
621 unsigned long start_time
= jiffies
;
624 while (!list_empty(&wb
->b_io
)) {
625 struct inode
*inode
= wb_inode(wb
->b_io
.prev
);
626 struct super_block
*sb
= inode
->i_sb
;
628 if (!grab_super_passive(sb
)) {
630 * grab_super_passive() may fail consistently due to
631 * s_umount being grabbed by someone else. Don't use
632 * requeue_io() to avoid busy retrying the inode/sb.
634 redirty_tail(inode
, wb
);
637 wrote
+= writeback_sb_inodes(sb
, wb
, work
);
640 /* refer to the same tests at the end of writeback_sb_inodes */
642 if (time_is_before_jiffies(start_time
+ HZ
/ 10UL))
644 if (work
->nr_pages
<= 0)
648 /* Leave any unwritten inodes on b_io */
652 long writeback_inodes_wb(struct bdi_writeback
*wb
, long nr_pages
,
653 enum wb_reason reason
)
655 struct wb_writeback_work work
= {
656 .nr_pages
= nr_pages
,
657 .sync_mode
= WB_SYNC_NONE
,
662 spin_lock(&wb
->list_lock
);
663 if (list_empty(&wb
->b_io
))
665 __writeback_inodes_wb(wb
, &work
);
666 spin_unlock(&wb
->list_lock
);
668 return nr_pages
- work
.nr_pages
;
671 static bool over_bground_thresh(struct backing_dev_info
*bdi
)
673 unsigned long background_thresh
, dirty_thresh
;
675 global_dirty_limits(&background_thresh
, &dirty_thresh
);
677 if (global_page_state(NR_FILE_DIRTY
) +
678 global_page_state(NR_UNSTABLE_NFS
) > background_thresh
)
681 if (bdi_stat(bdi
, BDI_RECLAIMABLE
) >
682 bdi_dirty_limit(bdi
, background_thresh
))
689 * Called under wb->list_lock. If there are multiple wb per bdi,
690 * only the flusher working on the first wb should do it.
692 static void wb_update_bandwidth(struct bdi_writeback
*wb
,
693 unsigned long start_time
)
695 __bdi_update_bandwidth(wb
->bdi
, 0, 0, 0, 0, 0, start_time
);
699 * Explicit flushing or periodic writeback of "old" data.
701 * Define "old": the first time one of an inode's pages is dirtied, we mark the
702 * dirtying-time in the inode's address_space. So this periodic writeback code
703 * just walks the superblock inode list, writing back any inodes which are
704 * older than a specific point in time.
706 * Try to run once per dirty_writeback_interval. But if a writeback event
707 * takes longer than a dirty_writeback_interval interval, then leave a
710 * older_than_this takes precedence over nr_to_write. So we'll only write back
711 * all dirty pages if they are all attached to "old" mappings.
713 static long wb_writeback(struct bdi_writeback
*wb
,
714 struct wb_writeback_work
*work
)
716 unsigned long wb_start
= jiffies
;
717 long nr_pages
= work
->nr_pages
;
718 unsigned long oldest_jif
;
722 oldest_jif
= jiffies
;
723 work
->older_than_this
= &oldest_jif
;
725 spin_lock(&wb
->list_lock
);
728 * Stop writeback when nr_pages has been consumed
730 if (work
->nr_pages
<= 0)
734 * Background writeout and kupdate-style writeback may
735 * run forever. Stop them if there is other work to do
736 * so that e.g. sync can proceed. They'll be restarted
737 * after the other works are all done.
739 if ((work
->for_background
|| work
->for_kupdate
) &&
740 !list_empty(&wb
->bdi
->work_list
))
744 * For background writeout, stop when we are below the
745 * background dirty threshold
747 if (work
->for_background
&& !over_bground_thresh(wb
->bdi
))
751 * Kupdate and background works are special and we want to
752 * include all inodes that need writing. Livelock avoidance is
753 * handled by these works yielding to any other work so we are
756 if (work
->for_kupdate
) {
757 oldest_jif
= jiffies
-
758 msecs_to_jiffies(dirty_expire_interval
* 10);
759 } else if (work
->for_background
)
760 oldest_jif
= jiffies
;
762 trace_writeback_start(wb
->bdi
, work
);
763 if (list_empty(&wb
->b_io
))
766 progress
= writeback_sb_inodes(work
->sb
, wb
, work
);
768 progress
= __writeback_inodes_wb(wb
, work
);
769 trace_writeback_written(wb
->bdi
, work
);
771 wb_update_bandwidth(wb
, wb_start
);
774 * Did we write something? Try for more
776 * Dirty inodes are moved to b_io for writeback in batches.
777 * The completion of the current batch does not necessarily
778 * mean the overall work is done. So we keep looping as long
779 * as made some progress on cleaning pages or inodes.
784 * No more inodes for IO, bail
786 if (list_empty(&wb
->b_more_io
))
789 * Nothing written. Wait for some inode to
790 * become available for writeback. Otherwise
791 * we'll just busyloop.
793 if (!list_empty(&wb
->b_more_io
)) {
794 trace_writeback_wait(wb
->bdi
, work
);
795 inode
= wb_inode(wb
->b_more_io
.prev
);
796 spin_lock(&inode
->i_lock
);
797 inode_wait_for_writeback(inode
, wb
);
798 spin_unlock(&inode
->i_lock
);
801 spin_unlock(&wb
->list_lock
);
803 return nr_pages
- work
->nr_pages
;
807 * Return the next wb_writeback_work struct that hasn't been processed yet.
809 static struct wb_writeback_work
*
810 get_next_work_item(struct backing_dev_info
*bdi
)
812 struct wb_writeback_work
*work
= NULL
;
814 spin_lock_bh(&bdi
->wb_lock
);
815 if (!list_empty(&bdi
->work_list
)) {
816 work
= list_entry(bdi
->work_list
.next
,
817 struct wb_writeback_work
, list
);
818 list_del_init(&work
->list
);
820 spin_unlock_bh(&bdi
->wb_lock
);
825 * Add in the number of potentially dirty inodes, because each inode
826 * write can dirty pagecache in the underlying blockdev.
828 static unsigned long get_nr_dirty_pages(void)
830 return global_page_state(NR_FILE_DIRTY
) +
831 global_page_state(NR_UNSTABLE_NFS
) +
832 get_nr_dirty_inodes();
835 static long wb_check_background_flush(struct bdi_writeback
*wb
)
837 if (over_bground_thresh(wb
->bdi
)) {
839 struct wb_writeback_work work
= {
840 .nr_pages
= LONG_MAX
,
841 .sync_mode
= WB_SYNC_NONE
,
844 .reason
= WB_REASON_BACKGROUND
,
847 return wb_writeback(wb
, &work
);
853 static long wb_check_old_data_flush(struct bdi_writeback
*wb
)
855 unsigned long expired
;
859 * When set to zero, disable periodic writeback
861 if (!dirty_writeback_interval
)
864 expired
= wb
->last_old_flush
+
865 msecs_to_jiffies(dirty_writeback_interval
* 10);
866 if (time_before(jiffies
, expired
))
869 wb
->last_old_flush
= jiffies
;
870 nr_pages
= get_nr_dirty_pages();
873 struct wb_writeback_work work
= {
874 .nr_pages
= nr_pages
,
875 .sync_mode
= WB_SYNC_NONE
,
878 .reason
= WB_REASON_PERIODIC
,
881 return wb_writeback(wb
, &work
);
888 * Retrieve work items and do the writeback they describe
890 long wb_do_writeback(struct bdi_writeback
*wb
, int force_wait
)
892 struct backing_dev_info
*bdi
= wb
->bdi
;
893 struct wb_writeback_work
*work
;
896 set_bit(BDI_writeback_running
, &wb
->bdi
->state
);
897 while ((work
= get_next_work_item(bdi
)) != NULL
) {
899 * Override sync mode, in case we must wait for completion
900 * because this thread is exiting now.
903 work
->sync_mode
= WB_SYNC_ALL
;
905 trace_writeback_exec(bdi
, work
);
907 wrote
+= wb_writeback(wb
, work
);
910 * Notify the caller of completion if this is a synchronous
911 * work item, otherwise just free it.
914 complete(work
->done
);
920 * Check for periodic writeback, kupdated() style
922 wrote
+= wb_check_old_data_flush(wb
);
923 wrote
+= wb_check_background_flush(wb
);
924 clear_bit(BDI_writeback_running
, &wb
->bdi
->state
);
930 * Handle writeback of dirty data for the device backed by this bdi. Also
931 * wakes up periodically and does kupdated style flushing.
933 int bdi_writeback_thread(void *data
)
935 struct bdi_writeback
*wb
= data
;
936 struct backing_dev_info
*bdi
= wb
->bdi
;
939 current
->flags
|= PF_SWAPWRITE
;
941 wb
->last_active
= jiffies
;
944 * Our parent may run at a different priority, just set us to normal
946 set_user_nice(current
, 0);
948 trace_writeback_thread_start(bdi
);
950 while (!kthread_freezable_should_stop(NULL
)) {
952 * Remove own delayed wake-up timer, since we are already awake
953 * and we'll take care of the preriodic write-back.
955 del_timer(&wb
->wakeup_timer
);
957 pages_written
= wb_do_writeback(wb
, 0);
959 trace_writeback_pages_written(pages_written
);
962 wb
->last_active
= jiffies
;
964 set_current_state(TASK_INTERRUPTIBLE
);
965 if (!list_empty(&bdi
->work_list
) || kthread_should_stop()) {
966 __set_current_state(TASK_RUNNING
);
970 if (wb_has_dirty_io(wb
) && dirty_writeback_interval
)
971 schedule_timeout(msecs_to_jiffies(dirty_writeback_interval
* 10));
974 * We have nothing to do, so can go sleep without any
975 * timeout and save power. When a work is queued or
976 * something is made dirty - we will be woken up.
982 /* Flush any work that raced with us exiting */
983 if (!list_empty(&bdi
->work_list
))
984 wb_do_writeback(wb
, 1);
986 trace_writeback_thread_stop(bdi
);
992 * Start writeback of `nr_pages' pages. If `nr_pages' is zero, write back
995 void wakeup_flusher_threads(long nr_pages
, enum wb_reason reason
)
997 struct backing_dev_info
*bdi
;
1000 nr_pages
= global_page_state(NR_FILE_DIRTY
) +
1001 global_page_state(NR_UNSTABLE_NFS
);
1005 list_for_each_entry_rcu(bdi
, &bdi_list
, bdi_list
) {
1006 if (!bdi_has_dirty_io(bdi
))
1008 __bdi_start_writeback(bdi
, nr_pages
, false, reason
);
1013 static noinline
void block_dump___mark_inode_dirty(struct inode
*inode
)
1015 if (inode
->i_ino
|| strcmp(inode
->i_sb
->s_id
, "bdev")) {
1016 struct dentry
*dentry
;
1017 const char *name
= "?";
1019 dentry
= d_find_alias(inode
);
1021 spin_lock(&dentry
->d_lock
);
1022 name
= (const char *) dentry
->d_name
.name
;
1025 "%s(%d): dirtied inode %lu (%s) on %s\n",
1026 current
->comm
, task_pid_nr(current
), inode
->i_ino
,
1027 name
, inode
->i_sb
->s_id
);
1029 spin_unlock(&dentry
->d_lock
);
1036 * __mark_inode_dirty - internal function
1037 * @inode: inode to mark
1038 * @flags: what kind of dirty (i.e. I_DIRTY_SYNC)
1039 * Mark an inode as dirty. Callers should use mark_inode_dirty or
1040 * mark_inode_dirty_sync.
1042 * Put the inode on the super block's dirty list.
1044 * CAREFUL! We mark it dirty unconditionally, but move it onto the
1045 * dirty list only if it is hashed or if it refers to a blockdev.
1046 * If it was not hashed, it will never be added to the dirty list
1047 * even if it is later hashed, as it will have been marked dirty already.
1049 * In short, make sure you hash any inodes _before_ you start marking
1052 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
1053 * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of
1054 * the kernel-internal blockdev inode represents the dirtying time of the
1055 * blockdev's pages. This is why for I_DIRTY_PAGES we always use
1056 * page->mapping->host, so the page-dirtying time is recorded in the internal
1059 void __mark_inode_dirty(struct inode
*inode
, int flags
)
1061 struct super_block
*sb
= inode
->i_sb
;
1062 struct backing_dev_info
*bdi
= NULL
;
1065 * Don't do this for I_DIRTY_PAGES - that doesn't actually
1066 * dirty the inode itself
1068 if (flags
& (I_DIRTY_SYNC
| I_DIRTY_DATASYNC
)) {
1069 if (sb
->s_op
->dirty_inode
)
1070 sb
->s_op
->dirty_inode(inode
, flags
);
1074 * make sure that changes are seen by all cpus before we test i_state
1079 /* avoid the locking if we can */
1080 if ((inode
->i_state
& flags
) == flags
)
1083 if (unlikely(block_dump
))
1084 block_dump___mark_inode_dirty(inode
);
1086 spin_lock(&inode
->i_lock
);
1087 if ((inode
->i_state
& flags
) != flags
) {
1088 const int was_dirty
= inode
->i_state
& I_DIRTY
;
1090 inode
->i_state
|= flags
;
1093 * If the inode is being synced, just update its dirty state.
1094 * The unlocker will place the inode on the appropriate
1095 * superblock list, based upon its state.
1097 if (inode
->i_state
& I_SYNC
)
1098 goto out_unlock_inode
;
1101 * Only add valid (hashed) inodes to the superblock's
1102 * dirty list. Add blockdev inodes as well.
1104 if (!S_ISBLK(inode
->i_mode
)) {
1105 if (inode_unhashed(inode
))
1106 goto out_unlock_inode
;
1108 if (inode
->i_state
& I_FREEING
)
1109 goto out_unlock_inode
;
1112 * If the inode was already on b_dirty/b_io/b_more_io, don't
1113 * reposition it (that would break b_dirty time-ordering).
1116 bool wakeup_bdi
= false;
1117 bdi
= inode_to_bdi(inode
);
1119 if (bdi_cap_writeback_dirty(bdi
)) {
1120 WARN(!test_bit(BDI_registered
, &bdi
->state
),
1121 "bdi-%s not registered\n", bdi
->name
);
1124 * If this is the first dirty inode for this
1125 * bdi, we have to wake-up the corresponding
1126 * bdi thread to make sure background
1127 * write-back happens later.
1129 if (!wb_has_dirty_io(&bdi
->wb
))
1133 spin_unlock(&inode
->i_lock
);
1134 spin_lock(&bdi
->wb
.list_lock
);
1135 inode
->dirtied_when
= jiffies
;
1136 list_move(&inode
->i_wb_list
, &bdi
->wb
.b_dirty
);
1137 spin_unlock(&bdi
->wb
.list_lock
);
1140 bdi_wakeup_thread_delayed(bdi
);
1145 spin_unlock(&inode
->i_lock
);
1148 EXPORT_SYMBOL(__mark_inode_dirty
);
1150 static void wait_sb_inodes(struct super_block
*sb
)
1152 struct inode
*inode
, *old_inode
= NULL
;
1155 * We need to be protected against the filesystem going from
1156 * r/o to r/w or vice versa.
1158 WARN_ON(!rwsem_is_locked(&sb
->s_umount
));
1160 spin_lock(&inode_sb_list_lock
);
1163 * Data integrity sync. Must wait for all pages under writeback,
1164 * because there may have been pages dirtied before our sync
1165 * call, but which had writeout started before we write it out.
1166 * In which case, the inode may not be on the dirty list, but
1167 * we still have to wait for that writeout.
1169 list_for_each_entry(inode
, &sb
->s_inodes
, i_sb_list
) {
1170 struct address_space
*mapping
= inode
->i_mapping
;
1172 spin_lock(&inode
->i_lock
);
1173 if ((inode
->i_state
& (I_FREEING
|I_WILL_FREE
|I_NEW
)) ||
1174 (mapping
->nrpages
== 0)) {
1175 spin_unlock(&inode
->i_lock
);
1179 spin_unlock(&inode
->i_lock
);
1180 spin_unlock(&inode_sb_list_lock
);
1183 * We hold a reference to 'inode' so it couldn't have been
1184 * removed from s_inodes list while we dropped the
1185 * inode_sb_list_lock. We cannot iput the inode now as we can
1186 * be holding the last reference and we cannot iput it under
1187 * inode_sb_list_lock. So we keep the reference and iput it
1193 filemap_fdatawait(mapping
);
1197 spin_lock(&inode_sb_list_lock
);
1199 spin_unlock(&inode_sb_list_lock
);
1204 * writeback_inodes_sb_nr - writeback dirty inodes from given super_block
1205 * @sb: the superblock
1206 * @nr: the number of pages to write
1207 * @reason: reason why some writeback work initiated
1209 * Start writeback on some inodes on this super_block. No guarantees are made
1210 * on how many (if any) will be written, and this function does not wait
1211 * for IO completion of submitted IO.
1213 void writeback_inodes_sb_nr(struct super_block
*sb
,
1215 enum wb_reason reason
)
1217 DECLARE_COMPLETION_ONSTACK(done
);
1218 struct wb_writeback_work work
= {
1220 .sync_mode
= WB_SYNC_NONE
,
1221 .tagged_writepages
= 1,
1227 WARN_ON(!rwsem_is_locked(&sb
->s_umount
));
1228 bdi_queue_work(sb
->s_bdi
, &work
);
1229 wait_for_completion(&done
);
1231 EXPORT_SYMBOL(writeback_inodes_sb_nr
);
1234 * writeback_inodes_sb - writeback dirty inodes from given super_block
1235 * @sb: the superblock
1236 * @reason: reason why some writeback work was initiated
1238 * Start writeback on some inodes on this super_block. No guarantees are made
1239 * on how many (if any) will be written, and this function does not wait
1240 * for IO completion of submitted IO.
1242 void writeback_inodes_sb(struct super_block
*sb
, enum wb_reason reason
)
1244 return writeback_inodes_sb_nr(sb
, get_nr_dirty_pages(), reason
);
1246 EXPORT_SYMBOL(writeback_inodes_sb
);
1249 * writeback_inodes_sb_if_idle - start writeback if none underway
1250 * @sb: the superblock
1251 * @reason: reason why some writeback work was initiated
1253 * Invoke writeback_inodes_sb if no writeback is currently underway.
1254 * Returns 1 if writeback was started, 0 if not.
1256 int writeback_inodes_sb_if_idle(struct super_block
*sb
, enum wb_reason reason
)
1258 if (!writeback_in_progress(sb
->s_bdi
)) {
1259 down_read(&sb
->s_umount
);
1260 writeback_inodes_sb(sb
, reason
);
1261 up_read(&sb
->s_umount
);
1266 EXPORT_SYMBOL(writeback_inodes_sb_if_idle
);
1269 * writeback_inodes_sb_nr_if_idle - start writeback if none underway
1270 * @sb: the superblock
1271 * @nr: the number of pages to write
1272 * @reason: reason why some writeback work was initiated
1274 * Invoke writeback_inodes_sb if no writeback is currently underway.
1275 * Returns 1 if writeback was started, 0 if not.
1277 int writeback_inodes_sb_nr_if_idle(struct super_block
*sb
,
1279 enum wb_reason reason
)
1281 if (!writeback_in_progress(sb
->s_bdi
)) {
1282 down_read(&sb
->s_umount
);
1283 writeback_inodes_sb_nr(sb
, nr
, reason
);
1284 up_read(&sb
->s_umount
);
1289 EXPORT_SYMBOL(writeback_inodes_sb_nr_if_idle
);
1292 * sync_inodes_sb - sync sb inode pages
1293 * @sb: the superblock
1295 * This function writes and waits on any dirty inode belonging to this
1298 void sync_inodes_sb(struct super_block
*sb
)
1300 DECLARE_COMPLETION_ONSTACK(done
);
1301 struct wb_writeback_work work
= {
1303 .sync_mode
= WB_SYNC_ALL
,
1304 .nr_pages
= LONG_MAX
,
1307 .reason
= WB_REASON_SYNC
,
1310 WARN_ON(!rwsem_is_locked(&sb
->s_umount
));
1312 bdi_queue_work(sb
->s_bdi
, &work
);
1313 wait_for_completion(&done
);
1317 EXPORT_SYMBOL(sync_inodes_sb
);
1320 * write_inode_now - write an inode to disk
1321 * @inode: inode to write to disk
1322 * @sync: whether the write should be synchronous or not
1324 * This function commits an inode to disk immediately if it is dirty. This is
1325 * primarily needed by knfsd.
1327 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
1329 int write_inode_now(struct inode
*inode
, int sync
)
1331 struct bdi_writeback
*wb
= &inode_to_bdi(inode
)->wb
;
1333 struct writeback_control wbc
= {
1334 .nr_to_write
= LONG_MAX
,
1335 .sync_mode
= sync
? WB_SYNC_ALL
: WB_SYNC_NONE
,
1337 .range_end
= LLONG_MAX
,
1340 if (!mapping_cap_writeback_dirty(inode
->i_mapping
))
1341 wbc
.nr_to_write
= 0;
1344 spin_lock(&wb
->list_lock
);
1345 spin_lock(&inode
->i_lock
);
1346 ret
= writeback_single_inode(inode
, wb
, &wbc
);
1347 spin_unlock(&inode
->i_lock
);
1348 spin_unlock(&wb
->list_lock
);
1351 EXPORT_SYMBOL(write_inode_now
);
1354 * sync_inode - write an inode and its pages to disk.
1355 * @inode: the inode to sync
1356 * @wbc: controls the writeback mode
1358 * sync_inode() will write an inode and its pages to disk. It will also
1359 * correctly update the inode on its superblock's dirty inode lists and will
1360 * update inode->i_state.
1362 * The caller must have a ref on the inode.
1364 int sync_inode(struct inode
*inode
, struct writeback_control
*wbc
)
1366 struct bdi_writeback
*wb
= &inode_to_bdi(inode
)->wb
;
1369 spin_lock(&wb
->list_lock
);
1370 spin_lock(&inode
->i_lock
);
1371 ret
= writeback_single_inode(inode
, wb
, wbc
);
1372 spin_unlock(&inode
->i_lock
);
1373 spin_unlock(&wb
->list_lock
);
1376 EXPORT_SYMBOL(sync_inode
);
1379 * sync_inode_metadata - write an inode to disk
1380 * @inode: the inode to sync
1381 * @wait: wait for I/O to complete.
1383 * Write an inode to disk and adjust its dirty state after completion.
1385 * Note: only writes the actual inode, no associated data or other metadata.
1387 int sync_inode_metadata(struct inode
*inode
, int wait
)
1389 struct writeback_control wbc
= {
1390 .sync_mode
= wait
? WB_SYNC_ALL
: WB_SYNC_NONE
,
1391 .nr_to_write
= 0, /* metadata-only */
1394 return sync_inode(inode
, &wbc
);
1396 EXPORT_SYMBOL(sync_inode_metadata
);