Commit | Line | Data |
---|---|---|
6fa3eb70 S |
1 | /* |
2 | * kernel/power/tuxonice_bio.c | |
3 | * | |
4 | * Copyright (C) 2004-2010 Nigel Cunningham (nigel at tuxonice net) | |
5 | * | |
6 | * Distributed under GPLv2. | |
7 | * | |
8 | * This file contains block io functions for TuxOnIce. These are | |
9 | * used by the swapwriter and it is planned that they will also | |
10 | * be used by the NFSwriter. | |
11 | * | |
12 | */ | |
13 | ||
14 | #include <linux/blkdev.h> | |
15 | #include <linux/syscalls.h> | |
16 | #include <linux/suspend.h> | |
17 | #include <linux/ctype.h> | |
18 | #include <linux/fs_uuid.h> | |
19 | ||
20 | #include "tuxonice.h" | |
21 | #include "tuxonice_sysfs.h" | |
22 | #include "tuxonice_modules.h" | |
23 | #include "tuxonice_prepare_image.h" | |
24 | #include "tuxonice_bio.h" | |
25 | #include "tuxonice_ui.h" | |
26 | #include "tuxonice_alloc.h" | |
27 | #include "tuxonice_io.h" | |
28 | #include "tuxonice_builtin.h" | |
29 | #include "tuxonice_bio_internal.h" | |
30 | ||
31 | #define MEMORY_ONLY 1 | |
32 | #define THROTTLE_WAIT 2 | |
33 | ||
34 | /* #define MEASURE_MUTEX_CONTENTION */ | |
35 | #ifndef MEASURE_MUTEX_CONTENTION | |
36 | #define my_mutex_lock(index, the_lock) mutex_lock(the_lock) | |
37 | #define my_mutex_unlock(index, the_lock) mutex_unlock(the_lock) | |
38 | #else | |
39 | unsigned long mutex_times[2][2][NR_CPUS]; | |
40 | #define my_mutex_lock(index, the_lock) do { \ | |
41 | int have_mutex; \ | |
42 | have_mutex = mutex_trylock(the_lock); \ | |
43 | if (!have_mutex) { \ | |
44 | mutex_lock(the_lock); \ | |
45 | mutex_times[index][0][smp_processor_id()]++; \ | |
46 | } else { \ | |
47 | mutex_times[index][1][smp_processor_id()]++; \ | |
48 | } | |
49 | ||
50 | #define my_mutex_unlock(index, the_lock) \ | |
51 | mutex_unlock(the_lock); \ | |
52 | } while (0) | |
53 | #endif | |
54 | ||
55 | static int page_idx, reset_idx; | |
56 | ||
57 | static int target_outstanding_io = 1024; | |
58 | static int max_outstanding_writes, max_outstanding_reads; | |
59 | ||
60 | static struct page *bio_queue_head, *bio_queue_tail; | |
61 | static atomic_t toi_bio_queue_size; | |
62 | static DEFINE_SPINLOCK(bio_queue_lock); | |
63 | ||
64 | static int free_mem_throttle, throughput_throttle; | |
65 | int more_readahead = 1; | |
66 | static struct page *readahead_list_head, *readahead_list_tail; | |
67 | ||
68 | static struct page *waiting_on; | |
69 | ||
70 | static atomic_t toi_io_in_progress, toi_io_done; | |
71 | static DECLARE_WAIT_QUEUE_HEAD(num_in_progress_wait); | |
72 | ||
73 | int current_stream; | |
74 | /* Not static, so that the allocators can setup and complete | |
75 | * writing the header */ | |
76 | char *toi_writer_buffer; | |
77 | int toi_writer_buffer_posn; | |
78 | ||
79 | static DEFINE_MUTEX(toi_bio_mutex); | |
80 | static DEFINE_MUTEX(toi_bio_readahead_mutex); | |
81 | ||
82 | static struct task_struct *toi_queue_flusher; | |
83 | static int toi_bio_queue_flush_pages(int dedicated_thread); | |
84 | ||
85 | struct toi_module_ops toi_blockwriter_ops; | |
86 | ||
87 | #define TOTAL_OUTSTANDING_IO (atomic_read(&toi_io_in_progress) + \ | |
88 | atomic_read(&toi_bio_queue_size)) | |
89 | ||
90 | unsigned long raw_pages_allocd, header_pages_reserved; | |
91 | ||
92 | /** | |
93 | * set_free_mem_throttle - set the point where we pause to avoid oom. | |
94 | * | |
95 | * Initially, this value is zero, but when we first fail to allocate memory, | |
96 | * we set it (plus a buffer) and thereafter throttle i/o once that limit is | |
97 | * reached. | |
98 | **/ | |
99 | static void set_free_mem_throttle(void) | |
100 | { | |
101 | int new_throttle = nr_unallocated_buffer_pages() + 256; | |
102 | ||
103 | if (new_throttle > free_mem_throttle) | |
104 | free_mem_throttle = new_throttle; | |
105 | } | |
106 | ||
107 | #define NUM_REASONS 7 | |
108 | static atomic_t reasons[NUM_REASONS]; | |
109 | static char *reason_name[NUM_REASONS] = { | |
110 | "readahead not ready", | |
111 | "bio allocation", | |
112 | "synchronous I/O", | |
113 | "toi_bio_get_new_page", | |
114 | "memory low", | |
115 | "readahead buffer allocation", | |
116 | "throughput_throttle", | |
117 | }; | |
118 | ||
119 | /* User Specified Parameters. */ | |
120 | unsigned long resume_firstblock; | |
121 | dev_t resume_dev_t; | |
122 | struct block_device *resume_block_device; | |
123 | static atomic_t resume_bdev_open_count; | |
124 | ||
125 | struct block_device *header_block_device; | |
126 | ||
127 | /** | |
128 | * toi_open_bdev: Open a bdev at resume time. | |
129 | * | |
130 | * index: The swap index. May be MAX_SWAPFILES for the resume_dev_t | |
131 | * (the user can have resume= pointing at a swap partition/file that isn't | |
132 | * swapon'd when they hibernate. MAX_SWAPFILES+1 for the first page of the | |
133 | * header. It will be from a swap partition that was enabled when we hibernated, | |
134 | * but we don't know it's real index until we read that first page. | |
135 | * dev_t: The device major/minor. | |
136 | * display_errs: Whether to try to do this quietly. | |
137 | * | |
138 | * We stored a dev_t in the image header. Open the matching device without | |
139 | * requiring /dev/<whatever> in most cases and record the details needed | |
140 | * to close it later and avoid duplicating work. | |
141 | */ | |
142 | struct block_device *toi_open_bdev(char *uuid, dev_t default_device, int display_errs) | |
143 | { | |
144 | struct block_device *bdev; | |
145 | dev_t device = default_device; | |
146 | char buf[32]; | |
147 | int retried = 0; | |
148 | ||
149 | retry: | |
150 | if (uuid) { | |
151 | struct fs_info seek; | |
152 | strncpy((char *)&seek.uuid, uuid, 16); | |
153 | seek.dev_t = 0; | |
154 | seek.last_mount_size = 0; | |
155 | device = blk_lookup_fs_info(&seek); | |
156 | if (!device) { | |
157 | device = default_device; | |
158 | printk(KERN_DEBUG "Unable to resolve uuid. Falling back" " to dev_t.\n"); | |
159 | } else | |
160 | printk(KERN_DEBUG "Resolved uuid to device %s.\n", | |
161 | format_dev_t(buf, device)); | |
162 | } | |
163 | ||
164 | if (!device) { | |
165 | printk(KERN_ERR "TuxOnIce attempting to open a " "blank dev_t!\n"); | |
166 | dump_stack(); | |
167 | return NULL; | |
168 | } | |
169 | bdev = toi_open_by_devnum(device); | |
170 | ||
171 | if (IS_ERR(bdev) || !bdev) { | |
172 | if (!retried) { | |
173 | retried = 1; | |
174 | wait_for_device_probe(); | |
175 | goto retry; | |
176 | } | |
177 | if (display_errs) | |
178 | toi_early_boot_message(1, TOI_CONTINUE_REQ, | |
179 | "Failed to get access to block device " | |
180 | "\"%x\" (error %d).\n Maybe you need " | |
181 | "to run mknod and/or lvmsetup in an " | |
182 | "initrd/ramfs?", device, bdev); | |
183 | return ERR_PTR(-EINVAL); | |
184 | } | |
185 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "TuxOnIce got bdev %p for dev_t %x.", bdev, device); | |
186 | ||
187 | return bdev; | |
188 | } | |
189 | ||
190 | static void toi_bio_reserve_header_space(unsigned long request) | |
191 | { | |
192 | header_pages_reserved = request; | |
193 | } | |
194 | ||
195 | /** | |
196 | * do_bio_wait - wait for some TuxOnIce I/O to complete | |
197 | * @reason: The array index of the reason we're waiting. | |
198 | * | |
199 | * Wait for a particular page of I/O if we're after a particular page. | |
200 | * If we're not after a particular page, wait instead for all in flight | |
201 | * I/O to be completed or for us to have enough free memory to be able | |
202 | * to submit more I/O. | |
203 | * | |
204 | * If we wait, we also update our statistics regarding why we waited. | |
205 | **/ | |
206 | static void do_bio_wait(int reason) | |
207 | { | |
208 | struct page *was_waiting_on = waiting_on; | |
209 | ||
210 | /* On SMP, waiting_on can be reset, so we make a copy */ | |
211 | if (was_waiting_on) { | |
212 | wait_on_page_locked(was_waiting_on); | |
213 | atomic_inc(&reasons[reason]); | |
214 | } else { | |
215 | atomic_inc(&reasons[reason]); | |
216 | ||
217 | wait_event(num_in_progress_wait, | |
218 | !atomic_read(&toi_io_in_progress) || | |
219 | nr_unallocated_buffer_pages() > free_mem_throttle); | |
220 | } | |
221 | } | |
222 | ||
223 | /** | |
224 | * throttle_if_needed - wait for I/O completion if throttle points are reached | |
225 | * @flags: What to check and how to act. | |
226 | * | |
227 | * Check whether we need to wait for some I/O to complete. We always check | |
228 | * whether we have enough memory available, but may also (depending upon | |
229 | * @reason) check if the throughput throttle limit has been reached. | |
230 | **/ | |
231 | static int throttle_if_needed(int flags) | |
232 | { | |
233 | int free_pages = nr_unallocated_buffer_pages(); | |
234 | ||
235 | /* Getting low on memory and I/O is in progress? */ | |
236 | while (unlikely(free_pages < free_mem_throttle) && | |
237 | atomic_read(&toi_io_in_progress) && !test_result_state(TOI_ABORTED)) { | |
238 | if (!(flags & THROTTLE_WAIT)) | |
239 | return -ENOMEM; | |
240 | do_bio_wait(4); | |
241 | free_pages = nr_unallocated_buffer_pages(); | |
242 | } | |
243 | ||
244 | while (!(flags & MEMORY_ONLY) && throughput_throttle && | |
245 | TOTAL_OUTSTANDING_IO >= throughput_throttle && !test_result_state(TOI_ABORTED)) { | |
246 | int result = toi_bio_queue_flush_pages(0); | |
247 | if (result) | |
248 | return result; | |
249 | atomic_inc(&reasons[6]); | |
250 | wait_event(num_in_progress_wait, | |
251 | !atomic_read(&toi_io_in_progress) || | |
252 | TOTAL_OUTSTANDING_IO < throughput_throttle); | |
253 | } | |
254 | ||
255 | return 0; | |
256 | } | |
257 | ||
258 | /** | |
259 | * update_throughput_throttle - update the raw throughput throttle | |
260 | * @jif_index: The number of times this function has been called. | |
261 | * | |
262 | * This function is called four times per second by the core, and used to limit | |
263 | * the amount of I/O we submit at once, spreading out our waiting through the | |
264 | * whole job and letting userui get an opportunity to do its work. | |
265 | * | |
266 | * We don't start limiting I/O until 1/4s has gone so that we get a | |
267 | * decent sample for our initial limit, and keep updating it because | |
268 | * throughput may vary (on rotating media, eg) with our block number. | |
269 | * | |
270 | * We throttle to 1/10s worth of I/O. | |
271 | **/ | |
272 | static void update_throughput_throttle(int jif_index) | |
273 | { | |
274 | int done = atomic_read(&toi_io_done); | |
275 | throughput_throttle = done * 2 / 5 / jif_index; | |
276 | } | |
277 | ||
278 | /** | |
279 | * toi_finish_all_io - wait for all outstanding i/o to complete | |
280 | * | |
281 | * Flush any queued but unsubmitted I/O and wait for it all to complete. | |
282 | **/ | |
283 | static int toi_finish_all_io(void) | |
284 | { | |
285 | int result = toi_bio_queue_flush_pages(0); | |
286 | toi_bio_queue_flusher_should_finish = 1; | |
287 | wake_up(&toi_io_queue_flusher); | |
288 | wait_event(num_in_progress_wait, !TOTAL_OUTSTANDING_IO); | |
289 | return result; | |
290 | } | |
291 | ||
292 | /** | |
293 | * toi_end_bio - bio completion function. | |
294 | * @bio: bio that has completed. | |
295 | * @err: Error value. Yes, like end_swap_bio_read, we ignore it. | |
296 | * | |
297 | * Function called by the block driver from interrupt context when I/O is | |
298 | * completed. If we were writing the page, we want to free it and will have | |
299 | * set bio->bi_private to the parameter we should use in telling the page | |
300 | * allocation accounting code what the page was allocated for. If we're | |
301 | * reading the page, it will be in the singly linked list made from | |
302 | * page->private pointers. | |
303 | **/ | |
304 | static void toi_end_bio(struct bio *bio, int err) | |
305 | { | |
306 | struct page *page = bio->bi_io_vec[0].bv_page; | |
307 | ||
308 | /* hib_log("err %d flags 0x%08x\n", err, bio->bi_flags); */ | |
309 | if (!err) | |
310 | BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags)); | |
311 | ||
312 | unlock_page(page); | |
313 | bio_put(bio); | |
314 | ||
315 | if (waiting_on == page) | |
316 | waiting_on = NULL; | |
317 | ||
318 | put_page(page); | |
319 | ||
320 | if (bio->bi_private) | |
321 | toi__free_page((int)((unsigned long)bio->bi_private), page); | |
322 | ||
323 | bio_put(bio); | |
324 | ||
325 | atomic_dec(&toi_io_in_progress); | |
326 | atomic_inc(&toi_io_done); | |
327 | ||
328 | wake_up(&num_in_progress_wait); | |
329 | } | |
330 | ||
331 | /** | |
332 | * submit - submit BIO request | |
333 | * @writing: READ or WRITE. | |
334 | * @dev: The block device we're using. | |
335 | * @first_block: The first sector we're using. | |
336 | * @page: The page being used for I/O. | |
337 | * @free_group: If writing, the group that was used in allocating the page | |
338 | * and which will be used in freeing the page from the completion | |
339 | * routine. | |
340 | * | |
341 | * Based on Patrick Mochell's pmdisk code from long ago: "Straight from the | |
342 | * textbook - allocate and initialize the bio. If we're writing, make sure | |
343 | * the page is marked as dirty. Then submit it and carry on." | |
344 | * | |
345 | * If we're just testing the speed of our own code, we fake having done all | |
346 | * the hard work and all toi_end_bio immediately. | |
347 | **/ | |
348 | static int submit(int writing, struct block_device *dev, sector_t first_block, | |
349 | struct page *page, int free_group) | |
350 | { | |
351 | struct bio *bio = NULL; | |
352 | int cur_outstanding_io, result; | |
353 | ||
354 | /* | |
355 | * Shouldn't throttle if reading - can deadlock in the single | |
356 | * threaded case as pages are only freed when we use the | |
357 | * readahead. | |
358 | */ | |
359 | if (writing) { | |
360 | result = throttle_if_needed(MEMORY_ONLY | THROTTLE_WAIT); | |
361 | if (result) | |
362 | return result; | |
363 | } | |
364 | ||
365 | while (!bio) { | |
366 | bio = bio_alloc(TOI_ATOMIC_GFP, 1); | |
367 | if (!bio) { | |
368 | set_free_mem_throttle(); | |
369 | do_bio_wait(1); | |
370 | } | |
371 | } | |
372 | ||
373 | bio->bi_bdev = dev; | |
374 | bio->bi_sector = first_block; | |
375 | bio->bi_private = (void *)((unsigned long)free_group); | |
376 | bio->bi_end_io = toi_end_bio; | |
377 | bio->bi_flags |= (1 << BIO_TOI); | |
378 | ||
379 | if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { | |
380 | printk(KERN_DEBUG "ERROR: adding page to bio at %lld\n", | |
381 | (unsigned long long)first_block); | |
382 | bio_put(bio); | |
383 | return -EFAULT; | |
384 | } | |
385 | ||
386 | bio_get(bio); | |
387 | ||
388 | cur_outstanding_io = atomic_add_return(1, &toi_io_in_progress); | |
389 | if (writing) { | |
390 | if (cur_outstanding_io > max_outstanding_writes) | |
391 | max_outstanding_writes = cur_outstanding_io; | |
392 | } else { | |
393 | if (cur_outstanding_io > max_outstanding_reads) | |
394 | max_outstanding_reads = cur_outstanding_io; | |
395 | } | |
396 | ||
397 | ||
398 | /* Still read the header! */ | |
399 | if (unlikely(test_action_state(TOI_TEST_BIO) && writing)) { | |
400 | /* Fake having done the hard work */ | |
401 | set_bit(BIO_UPTODATE, &bio->bi_flags); | |
402 | toi_end_bio(bio, 0); | |
403 | } else | |
404 | submit_bio(writing | REQ_SYNC, bio); | |
405 | ||
406 | return 0; | |
407 | } | |
408 | ||
409 | /** | |
410 | * toi_do_io: Prepare to do some i/o on a page and submit or batch it. | |
411 | * | |
412 | * @writing: Whether reading or writing. | |
413 | * @bdev: The block device which we're using. | |
414 | * @block0: The first sector we're reading or writing. | |
415 | * @page: The page on which I/O is being done. | |
416 | * @readahead_index: If doing readahead, the index (reset this flag when done). | |
417 | * @syncio: Whether the i/o is being done synchronously. | |
418 | * | |
419 | * Prepare and start a read or write operation. | |
420 | * | |
421 | * Note that we always work with our own page. If writing, we might be given a | |
422 | * compression buffer that will immediately be used to start compressing the | |
423 | * next page. For reading, we do readahead and therefore don't know the final | |
424 | * address where the data needs to go. | |
425 | **/ | |
426 | int toi_do_io(int writing, struct block_device *bdev, long block0, | |
427 | struct page *page, int is_readahead, int syncio, int free_group) | |
428 | { | |
429 | page->private = 0; | |
430 | ||
431 | /* Do here so we don't race against toi_bio_get_next_page_read */ | |
432 | lock_page(page); | |
433 | ||
434 | if (is_readahead) { | |
435 | if (readahead_list_head) | |
436 | readahead_list_tail->private = (unsigned long)page; | |
437 | else | |
438 | readahead_list_head = page; | |
439 | ||
440 | readahead_list_tail = page; | |
441 | } | |
442 | ||
443 | /* Done before submitting to avoid races. */ | |
444 | if (syncio) | |
445 | waiting_on = page; | |
446 | ||
447 | /* Submit the page */ | |
448 | get_page(page); | |
449 | ||
450 | if (submit(writing, bdev, block0, page, free_group)) | |
451 | return -EFAULT; | |
452 | ||
453 | if (syncio) | |
454 | do_bio_wait(2); | |
455 | ||
456 | return 0; | |
457 | } | |
458 | ||
459 | /** | |
460 | * toi_bdev_page_io - simpler interface to do directly i/o on a single page | |
461 | * @writing: Whether reading or writing. | |
462 | * @bdev: Block device on which we're operating. | |
463 | * @pos: Sector at which page to read or write starts. | |
464 | * @page: Page to be read/written. | |
465 | * | |
466 | * A simple interface to submit a page of I/O and wait for its completion. | |
467 | * The caller must free the page used. | |
468 | **/ | |
469 | static int toi_bdev_page_io(int writing, struct block_device *bdev, long pos, struct page *page) | |
470 | { | |
471 | return toi_do_io(writing, bdev, pos, page, 0, 1, 0); | |
472 | } | |
473 | ||
474 | /** | |
475 | * toi_bio_memory_needed - report the amount of memory needed for block i/o | |
476 | * | |
477 | * We want to have at least enough memory so as to have target_outstanding_io | |
478 | * or more transactions on the fly at once. If we can do more, fine. | |
479 | **/ | |
480 | static int toi_bio_memory_needed(void) | |
481 | { | |
482 | return target_outstanding_io * (PAGE_SIZE + sizeof(struct request) + sizeof(struct bio)); | |
483 | } | |
484 | ||
485 | /** | |
486 | * toi_bio_print_debug_stats - put out debugging info in the buffer provided | |
487 | * @buffer: A buffer of size @size into which text should be placed. | |
488 | * @size: The size of @buffer. | |
489 | * | |
490 | * Fill a buffer with debugging info. This is used for both our debug_info sysfs | |
491 | * entry and for recording the same info in dmesg. | |
492 | **/ | |
493 | static int toi_bio_print_debug_stats(char *buffer, int size) | |
494 | { | |
495 | int len = 0; | |
496 | ||
497 | if (toiActiveAllocator != &toi_blockwriter_ops) { | |
498 | len = scnprintf(buffer, size, "- Block I/O inactive.\n"); | |
499 | return len; | |
500 | } | |
501 | ||
502 | len = scnprintf(buffer, size, "- Block I/O active.\n"); | |
503 | ||
504 | len += toi_bio_chains_debug_info(buffer + len, size - len); | |
505 | ||
506 | len += scnprintf(buffer + len, size - len, | |
507 | "- Max outstanding reads %d. Max writes %d.\n", | |
508 | max_outstanding_reads, max_outstanding_writes); | |
509 | ||
510 | len += scnprintf(buffer + len, size - len, | |
511 | " Memory_needed: %d x (%lu + %u + %u) = %d bytes.\n", | |
512 | target_outstanding_io, | |
513 | PAGE_SIZE, (unsigned int)sizeof(struct request), | |
514 | (unsigned int)sizeof(struct bio), toi_bio_memory_needed()); | |
515 | ||
516 | #ifdef MEASURE_MUTEX_CONTENTION | |
517 | { | |
518 | int i; | |
519 | ||
520 | len += scnprintf(buffer + len, size - len, | |
521 | " Mutex contention while reading:\n Contended Free\n"); | |
522 | ||
523 | for_each_online_cpu(i) | |
524 | len += scnprintf(buffer + len, size - len, | |
525 | " %9lu %9lu\n", mutex_times[0][0][i], mutex_times[0][1][i]); | |
526 | ||
527 | len += scnprintf(buffer + len, size - len, | |
528 | " Mutex contention while writing:\n Contended Free\n"); | |
529 | ||
530 | for_each_online_cpu(i) | |
531 | len += scnprintf(buffer + len, size - len, | |
532 | " %9lu %9lu\n", mutex_times[1][0][i], mutex_times[1][1][i]); | |
533 | ||
534 | } | |
535 | #endif | |
536 | ||
537 | return len + scnprintf(buffer + len, size - len, | |
538 | " Free mem throttle point reached %d.\n", free_mem_throttle); | |
539 | } | |
540 | ||
541 | static int total_header_bytes; | |
542 | static int unowned; | |
543 | ||
544 | void debug_broken_header(void) | |
545 | { | |
546 | printk(KERN_DEBUG "Image header too big for size allocated!\n"); | |
547 | print_toi_header_storage_for_modules(); | |
548 | printk(KERN_DEBUG "Page flags : %d.\n", toi_pageflags_space_needed()); | |
549 | printk(KERN_DEBUG "toi_header : %zu.\n", sizeof(struct toi_header)); | |
550 | printk(KERN_DEBUG "Total unowned : %d.\n", unowned); | |
551 | printk(KERN_DEBUG "Total used : %d (%ld pages).\n", total_header_bytes, | |
552 | DIV_ROUND_UP(total_header_bytes, PAGE_SIZE)); | |
553 | printk(KERN_DEBUG "Space needed now : %ld.\n", get_header_storage_needed()); | |
554 | dump_block_chains(); | |
555 | abort_hibernate(TOI_HEADER_TOO_BIG, "Header reservation too small."); | |
556 | } | |
557 | ||
558 | /** | |
559 | * toi_rw_init - prepare to read or write a stream in the image | |
560 | * @writing: Whether reading or writing. | |
561 | * @stream number: Section of the image being processed. | |
562 | * | |
563 | * Prepare to read or write a section ('stream') in the image. | |
564 | **/ | |
565 | static int toi_rw_init(int writing, int stream_number) | |
566 | { | |
567 | if (stream_number) | |
568 | toi_extent_state_restore(stream_number); | |
569 | else | |
570 | toi_extent_state_goto_start(); | |
571 | ||
572 | if (writing) { | |
573 | reset_idx = 0; | |
574 | if (!current_stream) | |
575 | page_idx = 0; | |
576 | } else { | |
577 | reset_idx = 1; | |
578 | } | |
579 | ||
580 | atomic_set(&toi_io_done, 0); | |
581 | if (!toi_writer_buffer) | |
582 | toi_writer_buffer = (char *)toi_get_zeroed_page(11, TOI_ATOMIC_GFP); | |
583 | toi_writer_buffer_posn = writing ? 0 : PAGE_SIZE; | |
584 | ||
585 | current_stream = stream_number; | |
586 | ||
587 | more_readahead = 1; | |
588 | ||
589 | return toi_writer_buffer ? 0 : -ENOMEM; | |
590 | } | |
591 | ||
592 | /** | |
593 | * toi_bio_queue_write - queue a page for writing | |
594 | * @full_buffer: Pointer to a page to be queued | |
595 | * | |
596 | * Add a page to the queue to be submitted. If we're the queue flusher, | |
597 | * we'll do this once we've dropped toi_bio_mutex, so other threads can | |
598 | * continue to submit I/O while we're on the slow path doing the actual | |
599 | * submission. | |
600 | **/ | |
601 | static void toi_bio_queue_write(char **full_buffer) | |
602 | { | |
603 | struct page *page = virt_to_page(*full_buffer); | |
604 | unsigned long flags; | |
605 | ||
606 | *full_buffer = NULL; | |
607 | page->private = 0; | |
608 | ||
609 | spin_lock_irqsave(&bio_queue_lock, flags); | |
610 | if (!bio_queue_head) | |
611 | bio_queue_head = page; | |
612 | else | |
613 | bio_queue_tail->private = (unsigned long)page; | |
614 | ||
615 | bio_queue_tail = page; | |
616 | atomic_inc(&toi_bio_queue_size); | |
617 | ||
618 | spin_unlock_irqrestore(&bio_queue_lock, flags); | |
619 | wake_up(&toi_io_queue_flusher); | |
620 | } | |
621 | ||
622 | /** | |
623 | * toi_rw_cleanup - Cleanup after i/o. | |
624 | * @writing: Whether we were reading or writing. | |
625 | * | |
626 | * Flush all I/O and clean everything up after reading or writing a | |
627 | * section of the image. | |
628 | **/ | |
629 | static int toi_rw_cleanup(int writing) | |
630 | { | |
631 | int i, result = 0; | |
632 | ||
633 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_rw_cleanup."); | |
634 | if (writing) { | |
635 | if (toi_writer_buffer_posn && !test_result_state(TOI_ABORTED)) | |
636 | toi_bio_queue_write(&toi_writer_buffer); | |
637 | ||
638 | while (bio_queue_head && !result) | |
639 | result = toi_bio_queue_flush_pages(0); | |
640 | ||
641 | if (result) | |
642 | return result; | |
643 | ||
644 | if (current_stream == 2) | |
645 | toi_extent_state_save(1); | |
646 | else if (current_stream == 1) | |
647 | toi_extent_state_save(3); | |
648 | } | |
649 | ||
650 | result = toi_finish_all_io(); | |
651 | ||
652 | while (readahead_list_head) { | |
653 | void *next = (void *)readahead_list_head->private; | |
654 | toi__free_page(12, readahead_list_head); | |
655 | readahead_list_head = next; | |
656 | } | |
657 | ||
658 | readahead_list_tail = NULL; | |
659 | ||
660 | if (!current_stream) | |
661 | return result; | |
662 | ||
663 | for (i = 0; i < NUM_REASONS; i++) { | |
664 | if (!atomic_read(&reasons[i])) | |
665 | continue; | |
666 | printk(KERN_DEBUG "Waited for i/o due to %s %d times.\n", | |
667 | reason_name[i], atomic_read(&reasons[i])); | |
668 | atomic_set(&reasons[i], 0); | |
669 | } | |
670 | ||
671 | current_stream = 0; | |
672 | return result; | |
673 | } | |
674 | ||
675 | #ifdef CONFIG_TOI_FIXUP | |
676 | static void check_mem_status(void) | |
677 | { | |
678 | return; // supress messages, unless we need it!! | |
679 | hib_warn("%d: free/throttle: %lu/%d, no_readahead/target_oustanding_io:%d/%d io_in/io_done:%d/%d\n", __LINE__, | |
680 | nr_unallocated_buffer_pages(), free_mem_throttle , | |
681 | test_action_state(TOI_NO_READAHEAD), target_outstanding_io, | |
682 | atomic_read(&toi_io_in_progress), atomic_read(&toi_io_done)); | |
683 | HIB_SHOW_MEMINFO(); | |
684 | } | |
685 | ||
686 | static int hit_lowmem_barrier(void) | |
687 | { | |
688 | unsigned long free_pages; | |
689 | struct zone *zone; | |
690 | ||
691 | for_each_populated_zone(zone) { | |
692 | if (!strcmp(zone->name, "Normal")) { | |
693 | free_pages = zone_page_state(zone, NR_FREE_PAGES); | |
694 | return (free_pages < min_wmark_pages(zone)) ? 1 : 0; | |
695 | } | |
696 | } | |
697 | return 0; | |
698 | } | |
699 | #endif | |
700 | ||
701 | /** | |
702 | * toi_start_one_readahead - start one page of readahead | |
703 | * @dedicated_thread: Is this a thread dedicated to doing readahead? | |
704 | * | |
705 | * Start one new page of readahead. If this is being called by a thread | |
706 | * whose only just is to submit readahead, don't quit because we failed | |
707 | * to allocate a page. | |
708 | **/ | |
709 | static int toi_start_one_readahead(int dedicated_thread) | |
710 | { | |
711 | char *buffer = NULL; | |
712 | int oom = 0, result; | |
713 | ||
714 | result = throttle_if_needed(dedicated_thread ? THROTTLE_WAIT : 0); | |
715 | if (result) | |
716 | return result; | |
717 | ||
718 | mutex_lock(&toi_bio_readahead_mutex); | |
719 | ||
720 | while (!buffer) { | |
721 | #ifdef CONFIG_TOI_FIXUP | |
722 | buffer = likely(hit_lowmem_barrier()) ? NULL : (char *) toi_get_zeroed_page(12, TOI_ATOMIC_GFP); | |
723 | #else | |
724 | buffer = (char *) toi_get_zeroed_page(12, TOI_ATOMIC_GFP); | |
725 | #endif | |
726 | if (!buffer) { | |
727 | if (oom && !dedicated_thread) { | |
728 | mutex_unlock(&toi_bio_readahead_mutex); | |
729 | return -ENOMEM; | |
730 | } | |
731 | ||
732 | oom = 1; | |
733 | #ifdef CONFIG_TOI_FIXUP | |
734 | check_mem_status(); | |
735 | #endif | |
736 | set_free_mem_throttle(); | |
737 | do_bio_wait(5); | |
738 | } | |
739 | } | |
740 | ||
741 | result = toi_bio_rw_page(READ, virt_to_page(buffer), 1, 0); | |
742 | if (result == -ENOSPC) | |
743 | toi__free_page(12, virt_to_page(buffer)); | |
744 | mutex_unlock(&toi_bio_readahead_mutex); | |
745 | if (result) { | |
746 | if (result == -ENOSPC) | |
747 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "Last readahead page submitted."); | |
748 | else | |
749 | printk(KERN_DEBUG "toi_bio_rw_page returned %d.\n", result); | |
750 | } | |
751 | return result; | |
752 | } | |
753 | ||
754 | /** | |
755 | * toi_start_new_readahead - start new readahead | |
756 | * @dedicated_thread: Are we dedicated to this task? | |
757 | * | |
758 | * Start readahead of image pages. | |
759 | * | |
760 | * We can be called as a thread dedicated to this task (may be helpful on | |
761 | * systems with lots of CPUs), in which case we don't exit until there's no | |
762 | * more readahead. | |
763 | * | |
764 | * If this is not called by a dedicated thread, we top up our queue until | |
765 | * there's no more readahead to submit, we've submitted the number given | |
766 | * in target_outstanding_io or the number in progress exceeds the target | |
767 | * outstanding I/O value. | |
768 | * | |
769 | * No mutex needed because this is only ever called by the first cpu. | |
770 | **/ | |
771 | static int toi_start_new_readahead(int dedicated_thread) | |
772 | { | |
773 | int last_result, num_submitted = 0; | |
774 | ||
775 | /* Start a new readahead? */ | |
776 | if (!more_readahead) | |
777 | return 0; | |
778 | ||
779 | do { | |
780 | last_result = toi_start_one_readahead(dedicated_thread); | |
781 | ||
782 | if (last_result) { | |
783 | if (last_result == -ENOMEM || last_result == -ENOSPC) | |
784 | return 0; | |
785 | ||
786 | printk(KERN_DEBUG "Begin read chunk returned %d.\n", last_result); | |
787 | } else | |
788 | num_submitted++; | |
789 | ||
790 | } while (more_readahead && !last_result && | |
791 | (dedicated_thread || | |
792 | (num_submitted < target_outstanding_io && | |
793 | atomic_read(&toi_io_in_progress) < target_outstanding_io))); | |
794 | ||
795 | return last_result; | |
796 | } | |
797 | ||
798 | /** | |
799 | * bio_io_flusher - start the dedicated I/O flushing routine | |
800 | * @writing: Whether we're writing the image. | |
801 | **/ | |
802 | static int bio_io_flusher(int writing) | |
803 | { | |
804 | ||
805 | if (writing) | |
806 | return toi_bio_queue_flush_pages(1); | |
807 | else | |
808 | return toi_start_new_readahead(1); | |
809 | } | |
810 | ||
811 | /** | |
812 | * toi_bio_get_next_page_read - read a disk page, perhaps with readahead | |
813 | * @no_readahead: Whether we can use readahead | |
814 | * | |
815 | * Read a page from disk, submitting readahead and cleaning up finished i/o | |
816 | * while we wait for the page we're after. | |
817 | **/ | |
818 | static int toi_bio_get_next_page_read(int no_readahead) | |
819 | { | |
820 | char *virt; | |
821 | struct page *old_readahead_list_head; | |
822 | ||
823 | /* | |
824 | * When reading the second page of the header, we have to | |
825 | * delay submitting the read until after we've gotten the | |
826 | * extents out of the first page. | |
827 | */ | |
828 | if (unlikely(no_readahead && toi_start_one_readahead(0))) { | |
829 | printk(KERN_EMERG "%d:No readahead and toi_start_one_readahead " | |
830 | "returned non-zero.\n", __LINE__); | |
831 | return -EIO; | |
832 | } | |
833 | ||
834 | if (unlikely(!readahead_list_head)) { | |
835 | /* | |
836 | * If the last page finishes exactly on the page | |
837 | * boundary, we will be called one extra time and | |
838 | * have no data to return. In this case, we should | |
839 | * not BUG(), like we used to! | |
840 | */ | |
841 | if (!more_readahead) { | |
842 | printk(KERN_EMERG "No more readahead.\n"); | |
843 | return -ENOSPC; | |
844 | } | |
845 | if (unlikely(toi_start_one_readahead(0))) { | |
846 | printk(KERN_EMERG "%d:No readahead and " | |
847 | "toi_start_one_readahead returned non-zero.\n", __LINE__); | |
848 | return -EIO; | |
849 | } | |
850 | } | |
851 | ||
852 | if (PageLocked(readahead_list_head)) { | |
853 | waiting_on = readahead_list_head; | |
854 | do_bio_wait(0); | |
855 | } | |
856 | ||
857 | virt = page_address(readahead_list_head); | |
858 | memcpy(toi_writer_buffer, virt, PAGE_SIZE); | |
859 | ||
860 | mutex_lock(&toi_bio_readahead_mutex); | |
861 | old_readahead_list_head = readahead_list_head; | |
862 | readahead_list_head = (struct page *)readahead_list_head->private; | |
863 | mutex_unlock(&toi_bio_readahead_mutex); | |
864 | toi__free_page(12, old_readahead_list_head); | |
865 | return 0; | |
866 | } | |
867 | ||
868 | /** | |
869 | * toi_bio_queue_flush_pages - flush the queue of pages queued for writing | |
870 | * @dedicated_thread: Whether we're a dedicated thread | |
871 | * | |
872 | * Flush the queue of pages ready to be written to disk. | |
873 | * | |
874 | * If we're a dedicated thread, stay in here until told to leave, | |
875 | * sleeping in wait_event. | |
876 | * | |
877 | * The first thread is normally the only one to come in here. Another | |
878 | * thread can enter this routine too, though, via throttle_if_needed. | |
879 | * Since that's the case, we must be careful to only have one thread | |
880 | * doing this work at a time. Otherwise we have a race and could save | |
881 | * pages out of order. | |
882 | * | |
883 | * If an error occurs, free all remaining pages without submitting them | |
884 | * for I/O. | |
885 | **/ | |
886 | ||
887 | int toi_bio_queue_flush_pages(int dedicated_thread) | |
888 | { | |
889 | unsigned long flags; | |
890 | int result = 0; | |
891 | static DEFINE_MUTEX(busy); | |
892 | ||
893 | if (!mutex_trylock(&busy)) | |
894 | return 0; | |
895 | ||
896 | top: | |
897 | spin_lock_irqsave(&bio_queue_lock, flags); | |
898 | while (bio_queue_head) { | |
899 | struct page *page = bio_queue_head; | |
900 | bio_queue_head = (struct page *)page->private; | |
901 | if (bio_queue_tail == page) | |
902 | bio_queue_tail = NULL; | |
903 | atomic_dec(&toi_bio_queue_size); | |
904 | spin_unlock_irqrestore(&bio_queue_lock, flags); | |
905 | ||
906 | /* Don't generate more error messages if already had one */ | |
907 | if (!result) | |
908 | result = toi_bio_rw_page(WRITE, page, 0, 11); | |
909 | /* | |
910 | * If writing the page failed, don't drop out. | |
911 | * Flush the rest of the queue too. | |
912 | */ | |
913 | if (result) | |
914 | toi__free_page(11, page); | |
915 | spin_lock_irqsave(&bio_queue_lock, flags); | |
916 | } | |
917 | spin_unlock_irqrestore(&bio_queue_lock, flags); | |
918 | ||
919 | if (dedicated_thread) { | |
920 | wait_event(toi_io_queue_flusher, bio_queue_head || | |
921 | toi_bio_queue_flusher_should_finish); | |
922 | if (likely(!toi_bio_queue_flusher_should_finish)) | |
923 | goto top; | |
924 | toi_bio_queue_flusher_should_finish = 0; | |
925 | } | |
926 | ||
927 | mutex_unlock(&busy); | |
928 | return result; | |
929 | } | |
930 | ||
931 | /** | |
932 | * toi_bio_get_new_page - get a new page for I/O | |
933 | * @full_buffer: Pointer to a page to allocate. | |
934 | **/ | |
935 | static int toi_bio_get_new_page(char **full_buffer) | |
936 | { | |
937 | int result = throttle_if_needed(THROTTLE_WAIT); | |
938 | if (result) | |
939 | return result; | |
940 | ||
941 | while (!*full_buffer) { | |
942 | *full_buffer = (char *)toi_get_zeroed_page(11, TOI_ATOMIC_GFP); | |
943 | if (!*full_buffer) { | |
944 | set_free_mem_throttle(); | |
945 | do_bio_wait(3); | |
946 | } | |
947 | } | |
948 | ||
949 | return 0; | |
950 | } | |
951 | ||
952 | /** | |
953 | * toi_rw_buffer - combine smaller buffers into PAGE_SIZE I/O | |
954 | * @writing: Bool - whether writing (or reading). | |
955 | * @buffer: The start of the buffer to write or fill. | |
956 | * @buffer_size: The size of the buffer to write or fill. | |
957 | * @no_readahead: Don't try to start readhead (when getting extents). | |
958 | **/ | |
959 | static int toi_rw_buffer(int writing, char *buffer, int buffer_size, int no_readahead) | |
960 | { | |
961 | int bytes_left = buffer_size, result = 0; | |
962 | ||
963 | while (bytes_left) { | |
964 | char *source_start = buffer + buffer_size - bytes_left; | |
965 | char *dest_start = toi_writer_buffer + toi_writer_buffer_posn; | |
966 | int capacity = PAGE_SIZE - toi_writer_buffer_posn; | |
967 | char *to = writing ? dest_start : source_start; | |
968 | char *from = writing ? source_start : dest_start; | |
969 | ||
970 | if (bytes_left <= capacity) { | |
971 | memcpy(to, from, bytes_left); | |
972 | toi_writer_buffer_posn += bytes_left; | |
973 | return 0; | |
974 | } | |
975 | ||
976 | /* Complete this page and start a new one */ | |
977 | memcpy(to, from, capacity); | |
978 | bytes_left -= capacity; | |
979 | ||
980 | if (!writing) { | |
981 | /* | |
982 | * Perform actual I/O: | |
983 | * read readahead_list_head into toi_writer_buffer | |
984 | */ | |
985 | int result = toi_bio_get_next_page_read(no_readahead); | |
986 | if (result) { | |
987 | printk("toi_bio_get_next_page_read " "returned %d.\n", result); | |
988 | return result; | |
989 | } | |
990 | } else { | |
991 | toi_bio_queue_write(&toi_writer_buffer); | |
992 | result = toi_bio_get_new_page(&toi_writer_buffer); | |
993 | if (result) { | |
994 | printk(KERN_ERR "toi_bio_get_new_page returned " "%d.\n", result); | |
995 | return result; | |
996 | } | |
997 | } | |
998 | ||
999 | toi_writer_buffer_posn = 0; | |
1000 | toi_cond_pause(0, NULL); | |
1001 | } | |
1002 | ||
1003 | return 0; | |
1004 | } | |
1005 | ||
1006 | /** | |
1007 | * toi_bio_read_page - read a page of the image | |
1008 | * @pfn: The pfn where the data belongs. | |
1009 | * @buffer_page: The page containing the (possibly compressed) data. | |
1010 | * @buf_size: The number of bytes on @buffer_page used (PAGE_SIZE). | |
1011 | * | |
1012 | * Read a (possibly compressed) page from the image, into buffer_page, | |
1013 | * returning its pfn and the buffer size. | |
1014 | **/ | |
1015 | static int toi_bio_read_page(unsigned long *pfn, int buf_type, | |
1016 | void *buffer_page, unsigned int *buf_size) | |
1017 | { | |
1018 | int result = 0; | |
1019 | int this_idx; | |
1020 | char *buffer_virt = TOI_MAP(buf_type, buffer_page); | |
1021 | ||
1022 | /* | |
1023 | * Only call start_new_readahead if we don't have a dedicated thread | |
1024 | * and we're the queue flusher. | |
1025 | */ | |
1026 | if (current == toi_queue_flusher && more_readahead && !test_action_state(TOI_NO_READAHEAD)) { | |
1027 | int result2 = toi_start_new_readahead(0); | |
1028 | if (result2) { | |
1029 | printk(KERN_DEBUG "Queue flusher and " | |
1030 | "toi_start_one_readahead returned non-zero.\n"); | |
1031 | result = -EIO; | |
1032 | goto out; | |
1033 | } | |
1034 | } | |
1035 | ||
1036 | my_mutex_lock(0, &toi_bio_mutex); | |
1037 | ||
1038 | /* | |
1039 | * Structure in the image: | |
1040 | * [destination pfn|page size|page data] | |
1041 | * buf_size is PAGE_SIZE | |
1042 | * We can validly find there's nothing to read in a multithreaded | |
1043 | * situation. | |
1044 | */ | |
1045 | if (toi_rw_buffer(READ, (char *)&this_idx, sizeof(int), 0) || | |
1046 | toi_rw_buffer(READ, (char *)pfn, sizeof(unsigned long), 0) || | |
1047 | toi_rw_buffer(READ, (char *)buf_size, sizeof(int), 0) || | |
1048 | toi_rw_buffer(READ, buffer_virt, *buf_size, 0)) { | |
1049 | result = -ENODATA; | |
1050 | goto out_unlock; | |
1051 | } | |
1052 | ||
1053 | if (reset_idx) { | |
1054 | page_idx = this_idx; | |
1055 | reset_idx = 0; | |
1056 | } else { | |
1057 | page_idx++; | |
1058 | if (!this_idx) | |
1059 | result = -ENODATA; | |
1060 | else if (page_idx != this_idx) | |
1061 | printk(KERN_ERR "Got page index %d, expected %d.\n", this_idx, page_idx); | |
1062 | } | |
1063 | ||
1064 | out_unlock: | |
1065 | my_mutex_unlock(0, &toi_bio_mutex); | |
1066 | out: | |
1067 | TOI_UNMAP(buf_type, buffer_page); | |
1068 | return result; | |
1069 | } | |
1070 | ||
1071 | /** | |
1072 | * toi_bio_write_page - write a page of the image | |
1073 | * @pfn: The pfn where the data belongs. | |
1074 | * @buffer_page: The page containing the (possibly compressed) data. | |
1075 | * @buf_size: The number of bytes on @buffer_page used. | |
1076 | * | |
1077 | * Write a (possibly compressed) page to the image from the buffer, together | |
1078 | * with it's index and buffer size. | |
1079 | **/ | |
1080 | static int toi_bio_write_page(unsigned long pfn, int buf_type, | |
1081 | void *buffer_page, unsigned int buf_size) | |
1082 | { | |
1083 | char *buffer_virt; | |
1084 | int result = 0, result2 = 0; | |
1085 | ||
1086 | if (unlikely(test_action_state(TOI_TEST_FILTER_SPEED))) | |
1087 | return 0; | |
1088 | ||
1089 | my_mutex_lock(1, &toi_bio_mutex); | |
1090 | ||
1091 | if (test_result_state(TOI_ABORTED)) { | |
1092 | my_mutex_unlock(1, &toi_bio_mutex); | |
1093 | return 0; | |
1094 | } | |
1095 | ||
1096 | buffer_virt = TOI_MAP(buf_type, buffer_page); | |
1097 | page_idx++; | |
1098 | ||
1099 | /* | |
1100 | * Structure in the image: | |
1101 | * [destination pfn|page size|page data] | |
1102 | * buf_size is PAGE_SIZE | |
1103 | */ | |
1104 | if (toi_rw_buffer(WRITE, (char *)&page_idx, sizeof(int), 0) || | |
1105 | toi_rw_buffer(WRITE, (char *)&pfn, sizeof(unsigned long), 0) || | |
1106 | toi_rw_buffer(WRITE, (char *)&buf_size, sizeof(int), 0) || | |
1107 | toi_rw_buffer(WRITE, buffer_virt, buf_size, 0)) { | |
1108 | printk(KERN_DEBUG "toi_rw_buffer returned non-zero to " "toi_bio_write_page.\n"); | |
1109 | result = -EIO; | |
1110 | } | |
1111 | ||
1112 | TOI_UNMAP(buf_type, buffer_page); | |
1113 | my_mutex_unlock(1, &toi_bio_mutex); | |
1114 | ||
1115 | if (current == toi_queue_flusher) | |
1116 | result2 = toi_bio_queue_flush_pages(0); | |
1117 | ||
1118 | return result ? result : result2; | |
1119 | } | |
1120 | ||
1121 | /** | |
1122 | * _toi_rw_header_chunk - read or write a portion of the image header | |
1123 | * @writing: Whether reading or writing. | |
1124 | * @owner: The module for which we're writing. | |
1125 | * Used for confirming that modules | |
1126 | * don't use more header space than they asked for. | |
1127 | * @buffer: Address of the data to write. | |
1128 | * @buffer_size: Size of the data buffer. | |
1129 | * @no_readahead: Don't try to start readhead (when getting extents). | |
1130 | * | |
1131 | * Perform PAGE_SIZE I/O. Start readahead if needed. | |
1132 | **/ | |
1133 | static int _toi_rw_header_chunk(int writing, struct toi_module_ops *owner, | |
1134 | char *buffer, int buffer_size, int no_readahead) | |
1135 | { | |
1136 | int result = 0; | |
1137 | ||
1138 | if (owner) { | |
1139 | owner->header_used += buffer_size; | |
1140 | toi_message(TOI_HEADER, TOI_LOW, 1, | |
1141 | "Header: %s : %d bytes (%d/%d) from offset %d.", | |
1142 | owner->name, | |
1143 | buffer_size, owner->header_used, | |
1144 | owner->header_requested, toi_writer_buffer_posn); | |
1145 | if (owner->header_used > owner->header_requested && writing) { | |
1146 | printk(KERN_EMERG "TuxOnIce module %s is using more " | |
1147 | "header space (%u) than it requested (%u).\n", | |
1148 | owner->name, owner->header_used, owner->header_requested); | |
1149 | return buffer_size; | |
1150 | } | |
1151 | } else { | |
1152 | unowned += buffer_size; | |
1153 | toi_message(TOI_HEADER, TOI_LOW, 1, | |
1154 | "Header: (No owner): %d bytes (%d total so far) from " | |
1155 | "offset %d.", buffer_size, unowned, toi_writer_buffer_posn); | |
1156 | } | |
1157 | ||
1158 | if (!writing && !no_readahead && more_readahead) { | |
1159 | result = toi_start_new_readahead(0); | |
1160 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "Start new readahead " "returned %d.", result); | |
1161 | } | |
1162 | ||
1163 | if (!result) { | |
1164 | result = toi_rw_buffer(writing, buffer, buffer_size, no_readahead); | |
1165 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "rw_buffer returned " "%d.", result); | |
1166 | } | |
1167 | ||
1168 | total_header_bytes += buffer_size; | |
1169 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "_toi_rw_header_chunk returning " "%d.", result); | |
1170 | return result; | |
1171 | } | |
1172 | ||
1173 | static int toi_rw_header_chunk(int writing, struct toi_module_ops *owner, char *buffer, int size) | |
1174 | { | |
1175 | return _toi_rw_header_chunk(writing, owner, buffer, size, 1); | |
1176 | } | |
1177 | ||
1178 | static int toi_rw_header_chunk_noreadahead(int writing, | |
1179 | struct toi_module_ops *owner, char *buffer, int size) | |
1180 | { | |
1181 | return _toi_rw_header_chunk(writing, owner, buffer, size, 1); | |
1182 | } | |
1183 | ||
1184 | /** | |
1185 | * toi_bio_storage_needed - get the amount of storage needed for my fns | |
1186 | **/ | |
1187 | static int toi_bio_storage_needed(void) | |
1188 | { | |
1189 | return sizeof(int) + PAGE_SIZE + toi_bio_devinfo_storage_needed(); | |
1190 | } | |
1191 | ||
1192 | /** | |
1193 | * toi_bio_save_config_info - save block I/O config to image header | |
1194 | * @buf: PAGE_SIZE'd buffer into which data should be saved. | |
1195 | **/ | |
1196 | static int toi_bio_save_config_info(char *buf) | |
1197 | { | |
1198 | int *ints = (int *)buf; | |
1199 | ints[0] = target_outstanding_io; | |
1200 | return sizeof(int); | |
1201 | } | |
1202 | ||
1203 | /** | |
1204 | * toi_bio_load_config_info - restore block I/O config | |
1205 | * @buf: Data to be reloaded. | |
1206 | * @size: Size of the buffer saved. | |
1207 | **/ | |
1208 | static void toi_bio_load_config_info(char *buf, int size) | |
1209 | { | |
1210 | int *ints = (int *)buf; | |
1211 | target_outstanding_io = ints[0]; | |
1212 | } | |
1213 | ||
1214 | void close_resume_dev_t(int force) | |
1215 | { | |
1216 | if (!resume_block_device) | |
1217 | return; | |
1218 | ||
1219 | if (force) | |
1220 | atomic_set(&resume_bdev_open_count, 0); | |
1221 | else | |
1222 | atomic_dec(&resume_bdev_open_count); | |
1223 | ||
1224 | if (!atomic_read(&resume_bdev_open_count)) { | |
1225 | toi_close_bdev(resume_block_device); | |
1226 | resume_block_device = NULL; | |
1227 | } | |
1228 | } | |
1229 | ||
1230 | int open_resume_dev_t(int force, int quiet) | |
1231 | { | |
1232 | if (force) { | |
1233 | close_resume_dev_t(1); | |
1234 | atomic_set(&resume_bdev_open_count, 1); | |
1235 | } else | |
1236 | atomic_inc(&resume_bdev_open_count); | |
1237 | ||
1238 | if (resume_block_device) | |
1239 | return 0; | |
1240 | ||
1241 | resume_block_device = toi_open_bdev(NULL, resume_dev_t, 0); | |
1242 | if (IS_ERR(resume_block_device)) { | |
1243 | if (!quiet) | |
1244 | toi_early_boot_message(1, TOI_CONTINUE_REQ, | |
1245 | "Failed to open device %x, where" | |
1246 | " the header should be found.", resume_dev_t); | |
1247 | resume_block_device = NULL; | |
1248 | atomic_set(&resume_bdev_open_count, 0); | |
1249 | return 1; | |
1250 | } | |
1251 | ||
1252 | return 0; | |
1253 | } | |
1254 | ||
1255 | /** | |
1256 | * toi_bio_initialise - initialise bio code at start of some action | |
1257 | * @starting_cycle: Whether starting a hibernation cycle, or just reading or | |
1258 | * writing a sysfs value. | |
1259 | **/ | |
1260 | static int toi_bio_initialise(int starting_cycle) | |
1261 | { | |
1262 | int result; | |
1263 | ||
1264 | if (!starting_cycle || !resume_dev_t) | |
1265 | return 0; | |
1266 | ||
1267 | max_outstanding_writes = 0; | |
1268 | max_outstanding_reads = 0; | |
1269 | current_stream = 0; | |
1270 | toi_queue_flusher = current; | |
1271 | #ifdef MEASURE_MUTEX_CONTENTION | |
1272 | { | |
1273 | int i, j, k; | |
1274 | ||
1275 | for (i = 0; i < 2; i++) | |
1276 | for (j = 0; j < 2; j++) | |
1277 | for_each_online_cpu(k) | |
1278 | mutex_times[i][j][k] = 0; | |
1279 | } | |
1280 | #endif | |
1281 | result = open_resume_dev_t(0, 1); | |
1282 | ||
1283 | if (result) | |
1284 | return result; | |
1285 | ||
1286 | return get_signature_page(); | |
1287 | } | |
1288 | ||
1289 | static unsigned long raw_to_real(unsigned long raw) | |
1290 | { | |
1291 | unsigned long extra; | |
1292 | ||
1293 | extra = (raw * (sizeof(unsigned long) + sizeof(int)) + | |
1294 | (PAGE_SIZE + sizeof(unsigned long) + sizeof(int) + 1)) / | |
1295 | (PAGE_SIZE + sizeof(unsigned long) + sizeof(int)); | |
1296 | ||
1297 | return raw > extra ? raw - extra : 0; | |
1298 | } | |
1299 | ||
1300 | static unsigned long toi_bio_storage_available(void) | |
1301 | { | |
1302 | unsigned long sum = 0; | |
1303 | struct toi_module_ops *this_module; | |
1304 | ||
1305 | list_for_each_entry(this_module, &toi_modules, module_list) { | |
1306 | if (!this_module->enabled || this_module->type != BIO_ALLOCATOR_MODULE) | |
1307 | continue; | |
1308 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "Seeking storage " | |
1309 | "available from %s.", this_module->name); | |
1310 | sum += this_module->bio_allocator_ops->storage_available(); | |
1311 | } | |
1312 | ||
1313 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "Total storage available is %lu " | |
1314 | "pages (%d header pages).", sum, header_pages_reserved); | |
1315 | ||
1316 | return sum > header_pages_reserved ? raw_to_real(sum - header_pages_reserved) : 0; | |
1317 | ||
1318 | } | |
1319 | ||
1320 | static unsigned long toi_bio_storage_allocated(void) | |
1321 | { | |
1322 | return raw_pages_allocd > header_pages_reserved ? | |
1323 | raw_to_real(raw_pages_allocd - header_pages_reserved) : 0; | |
1324 | } | |
1325 | ||
1326 | /* | |
1327 | * If we have read part of the image, we might have filled memory with | |
1328 | * data that should be zeroed out. | |
1329 | */ | |
1330 | static void toi_bio_noresume_reset(void) | |
1331 | { | |
1332 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_noresume_reset."); | |
1333 | toi_rw_cleanup(READ); | |
1334 | free_all_bdev_info(); | |
1335 | } | |
1336 | ||
1337 | /** | |
1338 | * toi_bio_cleanup - cleanup after some action | |
1339 | * @finishing_cycle: Whether completing a cycle. | |
1340 | **/ | |
1341 | static void toi_bio_cleanup(int finishing_cycle) | |
1342 | { | |
1343 | if (!finishing_cycle) | |
1344 | return; | |
1345 | ||
1346 | if (toi_writer_buffer) { | |
1347 | toi_free_page(11, (unsigned long)toi_writer_buffer); | |
1348 | toi_writer_buffer = NULL; | |
1349 | } | |
1350 | ||
1351 | forget_signature_page(); | |
1352 | ||
1353 | if (header_block_device && toi_sig_data && toi_sig_data->header_dev_t != resume_dev_t) | |
1354 | toi_close_bdev(header_block_device); | |
1355 | ||
1356 | header_block_device = NULL; | |
1357 | ||
1358 | close_resume_dev_t(0); | |
1359 | } | |
1360 | ||
1361 | static int toi_bio_write_header_init(void) | |
1362 | { | |
1363 | int result; | |
1364 | ||
1365 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_write_header_init"); | |
1366 | toi_rw_init(WRITE, 0); | |
1367 | toi_writer_buffer_posn = 0; | |
1368 | ||
1369 | /* Info needed to bootstrap goes at the start of the header. | |
1370 | * First we save the positions and devinfo, including the number | |
1371 | * of header pages. Then we save the structs containing data needed | |
1372 | * for reading the header pages back. | |
1373 | * Note that even if header pages take more than one page, when we | |
1374 | * read back the info, we will have restored the location of the | |
1375 | * next header page by the time we go to use it. | |
1376 | */ | |
1377 | ||
1378 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "serialise extent chains."); | |
1379 | result = toi_serialise_extent_chains(); | |
1380 | ||
1381 | if (result) | |
1382 | return result; | |
1383 | ||
1384 | /* | |
1385 | * Signature page hasn't been modified at this point. Write it in | |
1386 | * the header so we can restore it later. | |
1387 | */ | |
1388 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "serialise signature page."); | |
1389 | return toi_rw_header_chunk_noreadahead(WRITE, &toi_blockwriter_ops, | |
1390 | (char *)toi_cur_sig_page, PAGE_SIZE); | |
1391 | } | |
1392 | ||
1393 | static int toi_bio_write_header_cleanup(void) | |
1394 | { | |
1395 | int result = 0; | |
1396 | ||
1397 | if (toi_writer_buffer_posn) | |
1398 | toi_bio_queue_write(&toi_writer_buffer); | |
1399 | ||
1400 | result = toi_finish_all_io(); | |
1401 | ||
1402 | unowned = 0; | |
1403 | total_header_bytes = 0; | |
1404 | ||
1405 | /* Set signature to save we have an image */ | |
1406 | if (!result) | |
1407 | result = toi_bio_mark_have_image(); | |
1408 | ||
1409 | return result; | |
1410 | } | |
1411 | ||
1412 | /* | |
1413 | * toi_bio_read_header_init() | |
1414 | * | |
1415 | * Description: | |
1416 | * 1. Attempt to read the device specified with resume=. | |
1417 | * 2. Check the contents of the swap header for our signature. | |
1418 | * 3. Warn, ignore, reset and/or continue as appropriate. | |
1419 | * 4. If continuing, read the toi_swap configuration section | |
1420 | * of the header and set up block device info so we can read | |
1421 | * the rest of the header & image. | |
1422 | * | |
1423 | * Returns: | |
1424 | * May not return if user choose to reboot at a warning. | |
1425 | * -EINVAL if cannot resume at this time. Booting should continue | |
1426 | * normally. | |
1427 | */ | |
1428 | ||
1429 | static int toi_bio_read_header_init(void) | |
1430 | { | |
1431 | int result = 0; | |
1432 | #if 0 /* JJ: mismatch caused by toi_bio_mark_have_image()'s wrong uuid information */ | |
1433 | char buf[32]; | |
1434 | #endif | |
1435 | ||
1436 | toi_writer_buffer_posn = 0; | |
1437 | ||
1438 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_read_header_init"); | |
1439 | ||
1440 | if (!toi_sig_data) { | |
1441 | printk(KERN_INFO "toi_bio_read_header_init called when we " | |
1442 | "haven't verified there is an image!\n"); | |
1443 | return -EINVAL; | |
1444 | } | |
1445 | ||
1446 | /* | |
1447 | * If the header is not on the resume_swap_dev_t, get the resume device | |
1448 | * first. | |
1449 | */ | |
1450 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "Header dev_t is %lx.", toi_sig_data->header_dev_t); | |
1451 | #if 0 /* JJ: mismatch caused by toi_bio_mark_have_image()'s wrong uuid information */ | |
1452 | if (toi_sig_data->have_uuid) { | |
1453 | struct fs_info seek; | |
1454 | dev_t device; | |
1455 | ||
1456 | strncpy((char *)seek.uuid, toi_sig_data->header_uuid, 16); | |
1457 | seek.dev_t = toi_sig_data->header_dev_t; | |
1458 | seek.last_mount_size = 0; | |
1459 | device = blk_lookup_fs_info(&seek); | |
1460 | if (device) { | |
1461 | printk("Using dev_t %s, returned by blk_lookup_fs_info.\n", | |
1462 | format_dev_t(buf, device)); | |
1463 | toi_sig_data->header_dev_t = device; | |
1464 | } | |
1465 | } | |
1466 | #endif | |
1467 | if (toi_sig_data->header_dev_t != resume_dev_t) { | |
1468 | header_block_device = toi_open_bdev(NULL, toi_sig_data->header_dev_t, 1); | |
1469 | ||
1470 | if (IS_ERR(header_block_device)) | |
1471 | return PTR_ERR(header_block_device); | |
1472 | } else | |
1473 | header_block_device = resume_block_device; | |
1474 | ||
1475 | if (!toi_writer_buffer) | |
1476 | toi_writer_buffer = (char *)toi_get_zeroed_page(11, TOI_ATOMIC_GFP); | |
1477 | more_readahead = 1; | |
1478 | ||
1479 | /* | |
1480 | * Read toi_swap configuration. | |
1481 | * Headerblock size taken into account already. | |
1482 | */ | |
1483 | result = toi_bio_ops.bdev_page_io(READ, header_block_device, | |
1484 | toi_sig_data->first_header_block, | |
1485 | virt_to_page((unsigned long)toi_writer_buffer)); | |
1486 | if (result) | |
1487 | return result; | |
1488 | ||
1489 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "load extent chains."); | |
1490 | result = toi_load_extent_chains(); | |
1491 | ||
1492 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "load original signature page."); | |
1493 | toi_orig_sig_page = (char *)toi_get_zeroed_page(38, TOI_ATOMIC_GFP); | |
1494 | if (!toi_orig_sig_page) { | |
1495 | printk(KERN_ERR "Failed to allocate memory for the current" " image signature.\n"); | |
1496 | return -ENOMEM; | |
1497 | } | |
1498 | ||
1499 | return toi_rw_header_chunk_noreadahead(READ, &toi_blockwriter_ops, | |
1500 | (char *)toi_orig_sig_page, PAGE_SIZE); | |
1501 | } | |
1502 | ||
1503 | static int toi_bio_read_header_cleanup(void) | |
1504 | { | |
1505 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_read_header_cleanup."); | |
1506 | return toi_rw_cleanup(READ); | |
1507 | } | |
1508 | ||
1509 | /* Works only for digits and letters, but small and fast */ | |
1510 | #define TOLOWER(x) ((x) | 0x20) | |
1511 | ||
1512 | /* | |
1513 | * UUID must be 32 chars long. It may have dashes, but nothing | |
1514 | * else. | |
1515 | */ | |
1516 | char *uuid_from_commandline(char *commandline) | |
1517 | { | |
1518 | int low = 0; | |
1519 | char *result = NULL, *output, *ptr; | |
1520 | ||
1521 | if (strncmp(commandline, "UUID=", 5)) | |
1522 | return NULL; | |
1523 | ||
1524 | result = kzalloc(17, GFP_KERNEL); | |
1525 | if (!result) { | |
1526 | printk("Failed to kzalloc UUID text memory.\n"); | |
1527 | return NULL; | |
1528 | } | |
1529 | ||
1530 | ptr = commandline + 5; | |
1531 | output = result; | |
1532 | ||
1533 | while (*ptr && (output - result) < 16) { | |
1534 | if (isxdigit(*ptr)) { | |
1535 | int value = isdigit(*ptr) ? *ptr - '0' : TOLOWER(*ptr) - 'a' + 10; | |
1536 | if (low) { | |
1537 | *output += value; | |
1538 | output++; | |
1539 | } else { | |
1540 | *output = value << 4; | |
1541 | } | |
1542 | low = !low; | |
1543 | } else if (*ptr != '-') | |
1544 | break; | |
1545 | ptr++; | |
1546 | } | |
1547 | ||
1548 | if ((output - result) < 16 || *ptr) { | |
1549 | printk(KERN_DEBUG "Found resume=UUID=, but the value looks " "invalid.\n"); | |
1550 | kfree(result); | |
1551 | result = NULL; | |
1552 | } | |
1553 | ||
1554 | return result; | |
1555 | } | |
1556 | ||
1557 | #define retry_if_fails(command) \ | |
1558 | do { \ | |
1559 | command; \ | |
1560 | if (!resume_dev_t && !waited_for_device_probe) { \ | |
1561 | wait_for_device_probe(); \ | |
1562 | command; \ | |
1563 | waited_for_device_probe = 1; \ | |
1564 | } \ | |
1565 | } while (0) | |
1566 | ||
1567 | /** | |
1568 | * try_to_open_resume_device: Try to parse and open resume= | |
1569 | * | |
1570 | * Any "swap:" has been stripped away and we just have the path to deal with. | |
1571 | * We attempt to do name_to_dev_t, open and stat the file. Having opened the | |
1572 | * file, get the struct block_device * to match. | |
1573 | */ | |
1574 | static int try_to_open_resume_device(char *commandline, int quiet) | |
1575 | { | |
1576 | struct kstat stat; | |
1577 | int error = 0; | |
1578 | char *uuid = uuid_from_commandline(commandline); | |
1579 | int waited_for_device_probe = 0; | |
1580 | ||
1581 | hib_log("commandline=\"%s\"\n", commandline); | |
1582 | resume_dev_t = MKDEV(0, 0); | |
1583 | ||
1584 | if (!strlen(commandline)) { | |
1585 | #ifdef CONFIG_TOI_FIXUP | |
1586 | hib_warn("skip scanning for image...\n"); | |
1587 | #else | |
1588 | retry_if_fails(toi_bio_scan_for_image(quiet)); | |
1589 | #endif | |
1590 | } | |
1591 | ||
1592 | if (uuid) { | |
1593 | struct fs_info seek; | |
1594 | strncpy((char *)&seek.uuid, uuid, 16); | |
1595 | seek.dev_t = resume_dev_t; | |
1596 | seek.last_mount_size = 0; | |
1597 | retry_if_fails(resume_dev_t = blk_lookup_fs_info(&seek)); | |
1598 | kfree(uuid); | |
1599 | } | |
1600 | ||
1601 | if (!resume_dev_t) | |
1602 | retry_if_fails(resume_dev_t = name_to_dev_t(commandline)); | |
1603 | ||
1604 | if (!resume_dev_t) { | |
1605 | struct file *file = filp_open(commandline, | |
1606 | O_RDONLY | O_LARGEFILE, 0); | |
1607 | ||
1608 | if (!IS_ERR(file) && file) { | |
1609 | vfs_getattr(&file->f_path, &stat); | |
1610 | filp_close(file, NULL); | |
1611 | } else | |
1612 | error = vfs_stat(commandline, &stat); | |
1613 | if (!error) | |
1614 | resume_dev_t = stat.rdev; | |
1615 | } | |
1616 | ||
1617 | if (!resume_dev_t) { | |
1618 | if (quiet) | |
1619 | return 1; | |
1620 | ||
1621 | if (test_toi_state(TOI_TRYING_TO_RESUME)) | |
1622 | toi_early_boot_message(1, toi_translate_err_default, | |
1623 | "Failed to translate \"%s\" into a device id.\n", | |
1624 | commandline); | |
1625 | else if (strlen(commandline)) | |
1626 | printk("TuxOnIce: Can't translate \"%s\" into a device " | |
1627 | "id yet.\n", commandline); | |
1628 | return 1; | |
1629 | } | |
1630 | ||
1631 | return open_resume_dev_t(1, quiet); | |
1632 | } | |
1633 | ||
1634 | /* | |
1635 | * Parse Image Location | |
1636 | * | |
1637 | * Attempt to parse a resume= parameter. | |
1638 | * Swap Writer accepts: | |
1639 | * resume=[swap:|file:]DEVNAME[:FIRSTBLOCK][@BLOCKSIZE] | |
1640 | * | |
1641 | * Where: | |
1642 | * DEVNAME is convertable to a dev_t by name_to_dev_t | |
1643 | * FIRSTBLOCK is the location of the first block in the swap file | |
1644 | * (specifying for a swap partition is nonsensical but not prohibited). | |
1645 | * Data is validated by attempting to read a swap header from the | |
1646 | * location given. Failure will result in toi_swap refusing to | |
1647 | * save an image, and a reboot with correct parameters will be | |
1648 | * necessary. | |
1649 | */ | |
1650 | static int toi_bio_parse_sig_location(char *commandline, int only_allocator, int quiet) | |
1651 | { | |
1652 | char *thischar, *devstart, *colon = NULL; | |
1653 | int signature_found, result = -EINVAL, temp_result = 0; | |
1654 | ||
1655 | if (strncmp(commandline, "swap:", 5) && strncmp(commandline, "file:", 5)) { | |
1656 | /* | |
1657 | * Failing swap:, we'll take a simple resume=/dev/hda2, or a | |
1658 | * blank value (scan) but fall through to other allocators | |
1659 | * if /dev/ or UUID= isn't matched. | |
1660 | */ | |
1661 | if (strncmp(commandline, "/dev/", 5) && | |
1662 | strncmp(commandline, "UUID=", 5) && strlen(commandline)) | |
1663 | return 1; | |
1664 | } else | |
1665 | commandline += 5; | |
1666 | ||
1667 | devstart = commandline; | |
1668 | thischar = commandline; | |
1669 | while ((*thischar != ':') && (*thischar != '@') && | |
1670 | ((thischar - commandline) < 250) && (*thischar)) | |
1671 | thischar++; | |
1672 | ||
1673 | if (*thischar == ':') { | |
1674 | colon = thischar; | |
1675 | *colon = 0; | |
1676 | thischar++; | |
1677 | } | |
1678 | ||
1679 | while ((thischar - commandline) < 250 && *thischar) | |
1680 | thischar++; | |
1681 | ||
1682 | if (colon) { | |
1683 | unsigned long block; | |
1684 | temp_result = strict_strtoul(colon + 1, 0, &block); | |
1685 | if (!temp_result) | |
1686 | resume_firstblock = (int)block; | |
1687 | } else if (swsusp_resume_block) { | |
1688 | resume_firstblock = swsusp_resume_block; | |
1689 | } else | |
1690 | resume_firstblock = 0; | |
1691 | ||
1692 | clear_toi_state(TOI_CAN_HIBERNATE); | |
1693 | clear_toi_state(TOI_CAN_RESUME); | |
1694 | ||
1695 | if (!temp_result) | |
1696 | temp_result = try_to_open_resume_device(devstart, quiet); | |
1697 | ||
1698 | if (colon) | |
1699 | *colon = ':'; | |
1700 | ||
1701 | /* No error if we only scanned */ | |
1702 | if (temp_result) | |
1703 | return strlen(commandline) ? -EINVAL : 1; | |
1704 | ||
1705 | signature_found = toi_bio_image_exists(quiet); | |
1706 | ||
1707 | if (signature_found != -1) { | |
1708 | result = 0; | |
1709 | /* | |
1710 | * TODO: If only file storage, CAN_HIBERNATE should only be | |
1711 | * set if file allocator's target is valid. | |
1712 | */ | |
1713 | set_toi_state(TOI_CAN_HIBERNATE); | |
1714 | set_toi_state(TOI_CAN_RESUME); | |
1715 | } else if (!quiet) | |
1716 | printk(KERN_ERR "TuxOnIce: Block I/O: No " "signature found at %s.\n", devstart); | |
1717 | ||
1718 | return result; | |
1719 | } | |
1720 | ||
1721 | static void toi_bio_release_storage(void) | |
1722 | { | |
1723 | header_pages_reserved = 0; | |
1724 | raw_pages_allocd = 0; | |
1725 | ||
1726 | free_all_bdev_info(); | |
1727 | } | |
1728 | ||
1729 | /* toi_swap_remove_image | |
1730 | * | |
1731 | */ | |
1732 | static int toi_bio_remove_image(void) | |
1733 | { | |
1734 | int result; | |
1735 | ||
1736 | toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_remove_image."); | |
1737 | ||
1738 | result = toi_bio_restore_original_signature(); | |
1739 | ||
1740 | /* | |
1741 | * We don't do a sanity check here: we want to restore the swap | |
1742 | * whatever version of kernel made the hibernate image. | |
1743 | * | |
1744 | * We need to write swap, but swap may not be enabled so | |
1745 | * we write the device directly | |
1746 | * | |
1747 | * If we don't have an current_signature_page, we didn't | |
1748 | * read an image header, so don't change anything. | |
1749 | */ | |
1750 | ||
1751 | toi_bio_release_storage(); | |
1752 | ||
1753 | return result; | |
1754 | } | |
1755 | ||
1756 | struct toi_bio_ops toi_bio_ops = { | |
1757 | .bdev_page_io = toi_bdev_page_io, | |
1758 | .register_storage = toi_register_storage_chain, | |
1759 | .free_storage = toi_bio_release_storage, | |
1760 | }; | |
1761 | EXPORT_SYMBOL_GPL(toi_bio_ops); | |
1762 | ||
1763 | static struct toi_sysfs_data sysfs_params[] = { | |
1764 | SYSFS_INT("target_outstanding_io", SYSFS_RW, &target_outstanding_io, | |
1765 | 0, 16384, 0, NULL), | |
1766 | }; | |
1767 | ||
1768 | struct toi_module_ops toi_blockwriter_ops = { | |
1769 | .type = WRITER_MODULE, | |
1770 | .name = "block i/o", | |
1771 | .directory = "block_io", | |
1772 | .module = THIS_MODULE, | |
1773 | .memory_needed = toi_bio_memory_needed, | |
1774 | .print_debug_info = toi_bio_print_debug_stats, | |
1775 | .storage_needed = toi_bio_storage_needed, | |
1776 | .save_config_info = toi_bio_save_config_info, | |
1777 | .load_config_info = toi_bio_load_config_info, | |
1778 | .initialise = toi_bio_initialise, | |
1779 | .cleanup = toi_bio_cleanup, | |
1780 | .post_atomic_restore = toi_bio_chains_post_atomic, | |
1781 | ||
1782 | .rw_init = toi_rw_init, | |
1783 | .rw_cleanup = toi_rw_cleanup, | |
1784 | .read_page = toi_bio_read_page, | |
1785 | .write_page = toi_bio_write_page, | |
1786 | .rw_header_chunk = toi_rw_header_chunk, | |
1787 | .rw_header_chunk_noreadahead = toi_rw_header_chunk_noreadahead, | |
1788 | .io_flusher = bio_io_flusher, | |
1789 | .update_throughput_throttle = update_throughput_throttle, | |
1790 | .finish_all_io = toi_finish_all_io, | |
1791 | ||
1792 | .noresume_reset = toi_bio_noresume_reset, | |
1793 | .storage_available = toi_bio_storage_available, | |
1794 | .storage_allocated = toi_bio_storage_allocated, | |
1795 | .reserve_header_space = toi_bio_reserve_header_space, | |
1796 | .allocate_storage = toi_bio_allocate_storage, | |
1797 | .image_exists = toi_bio_image_exists, | |
1798 | .mark_resume_attempted = toi_bio_mark_resume_attempted, | |
1799 | .write_header_init = toi_bio_write_header_init, | |
1800 | .write_header_cleanup = toi_bio_write_header_cleanup, | |
1801 | .read_header_init = toi_bio_read_header_init, | |
1802 | .read_header_cleanup = toi_bio_read_header_cleanup, | |
1803 | .get_header_version = toi_bio_get_header_version, | |
1804 | .remove_image = toi_bio_remove_image, | |
1805 | .parse_sig_location = toi_bio_parse_sig_location, | |
1806 | ||
1807 | .sysfs_data = sysfs_params, | |
1808 | .num_sysfs_entries = sizeof(sysfs_params) / sizeof(struct toi_sysfs_data), | |
1809 | }; | |
1810 | ||
1811 | /** | |
1812 | * toi_block_io_load - load time routine for block I/O module | |
1813 | * | |
1814 | * Register block i/o ops and sysfs entries. | |
1815 | **/ | |
1816 | static __init int toi_block_io_load(void) | |
1817 | { | |
1818 | return toi_register_module(&toi_blockwriter_ops); | |
1819 | } | |
1820 | ||
1821 | #ifdef MODULE | |
1822 | static __exit void toi_block_io_unload(void) | |
1823 | { | |
1824 | toi_unregister_module(&toi_blockwriter_ops); | |
1825 | } | |
1826 | module_init(toi_block_io_load); | |
1827 | module_exit(toi_block_io_unload); | |
1828 | MODULE_LICENSE("GPL"); | |
1829 | MODULE_AUTHOR("Nigel Cunningham"); | |
1830 | MODULE_DESCRIPTION("TuxOnIce block io functions"); | |
1831 | #else | |
1832 | late_initcall(toi_block_io_load); | |
1833 | #endif |