kernel/power/snapshot.c

   1 /*
   2  * linux/kernel/power/snapshot.c
   3  *
   4  * This file provide system snapshot/restore functionality.
   5  *
   6  * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
   7  *
   8  * This file is released under the GPLv2, and is based on swsusp.c.
   9  *
  10  */
  11
  12
  13 #include <linux/module.h>
  14 #include <linux/mm.h>
  15 #include <linux/suspend.h>
  16 #include <linux/smp_lock.h>
  17 #include <linux/delay.h>
  18 #include <linux/bitops.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/kernel.h>
  21 #include <linux/pm.h>
  22 #include <linux/device.h>
  23 #include <linux/bootmem.h>
  24 #include <linux/syscalls.h>
  25 #include <linux/console.h>
  26 #include <linux/highmem.h>
  27
  28 #include <asm/uaccess.h>
  29 #include <asm/mmu_context.h>
  30 #include <asm/pgtable.h>
  31 #include <asm/tlbflush.h>
  32 #include <asm/io.h>
  33
  34 #include "power.h"
  35
  36 #ifdef CONFIG_HIGHMEM
  37 struct highmem_page {
  38         char *data;
  39         struct page *page;
  40         struct highmem_page *next;
  41 };
  42
  43 static struct highmem_page *highmem_copy;
  44
  45 static int save_highmem_zone(struct zone *zone)
  46 {
  47         unsigned long zone_pfn;
  48         mark_free_pages(zone);
  49         for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
  50                 struct page *page;
  51                 struct highmem_page *save;
  52                 void *kaddr;
  53                 unsigned long pfn = zone_pfn + zone->zone_start_pfn;
  54
  55                 if (!(pfn%1000))
  56                         printk(".");
  57                 if (!pfn_valid(pfn))
  58                         continue;
  59                 page = pfn_to_page(pfn);
  60                 /*
  61                  * This condition results from rvmalloc() sans vmalloc_32()
  62                  * and architectural memory reservations. This should be
  63                  * corrected eventually when the cases giving rise to this
  64                  * are better understood.
  65                  */
  66                 if (PageReserved(page)) {
  67                         printk("highmem reserved page?!\n");
  68                         continue;
  69                 }
  70                 BUG_ON(PageNosave(page));
  71                 if (PageNosaveFree(page))
  72                         continue;
  73                 save = kmalloc(sizeof(struct highmem_page), GFP_ATOMIC);
  74                 if (!save)
  75                         return -ENOMEM;
  76                 save->next = highmem_copy;
  77                 save->page = page;
  78                 save->data = (void *) get_zeroed_page(GFP_ATOMIC);
  79                 if (!save->data) {
  80                         kfree(save);
  81                         return -ENOMEM;
  82                 }
  83                 kaddr = kmap_atomic(page, KM_USER0);
  84                 memcpy(save->data, kaddr, PAGE_SIZE);
  85                 kunmap_atomic(kaddr, KM_USER0);
  86                 highmem_copy = save;
  87         }
  88         return 0;
  89 }
  90
  91 int save_highmem(void)
  92 {
  93         struct zone *zone;
  94         int res = 0;
  95
  96         pr_debug("swsusp: Saving Highmem\n");
  97         for_each_zone (zone) {
  98                 if (is_highmem(zone))
  99                         res = save_highmem_zone(zone);
 100                 if (res)
 101                         return res;
 102         }
 103         return 0;
 104 }
 105
 106 int restore_highmem(void)
 107 {
 108         printk("swsusp: Restoring Highmem\n");
 109         while (highmem_copy) {
 110                 struct highmem_page *save = highmem_copy;
 111                 void *kaddr;
 112                 highmem_copy = save->next;
 113
 114                 kaddr = kmap_atomic(save->page, KM_USER0);
 115                 memcpy(kaddr, save->data, PAGE_SIZE);
 116                 kunmap_atomic(kaddr, KM_USER0);
 117                 free_page((long) save->data);
 118                 kfree(save);
 119         }
 120         return 0;
 121 }
 122 #endif
 123
 124 static int pfn_is_nosave(unsigned long pfn)
 125 {
 126         unsigned long nosave_begin_pfn = __pa(&__nosave_begin) >> PAGE_SHIFT;
 127         unsigned long nosave_end_pfn = PAGE_ALIGN(__pa(&__nosave_end)) >> PAGE_SHIFT;
 128         return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
 129 }
 130
 131 /**
 132  *      saveable - Determine whether a page should be cloned or not.
 133  *      @pfn:   The page
 134  *
 135  *      We save a page if it's Reserved, and not in the range of pages
 136  *      statically defined as 'unsaveable', or if it isn't reserved, and
 137  *      isn't part of a free chunk of pages.
 138  */
 139
 140 static int saveable(struct zone *zone, unsigned long *zone_pfn)
 141 {
 142         unsigned long pfn = *zone_pfn + zone->zone_start_pfn;
 143         struct page *page;
 144
 145         if (!pfn_valid(pfn))
 146                 return 0;
 147
 148         page = pfn_to_page(pfn);
 149         BUG_ON(PageReserved(page) && PageNosave(page));
 150         if (PageNosave(page))
 151                 return 0;
 152         if (PageReserved(page) && pfn_is_nosave(pfn)) {
 153                 pr_debug("[nosave pfn 0x%lx]", pfn);
 154                 return 0;
 155         }
 156         if (PageNosaveFree(page))
 157                 return 0;
 158
 159         return 1;
 160 }
 161
 162 static unsigned count_data_pages(void)
 163 {
 164         struct zone *zone;
 165         unsigned long zone_pfn;
 166         unsigned int n = 0;
 167
 168         for_each_zone (zone) {
 169                 if (is_highmem(zone))
 170                         continue;
 171                 mark_free_pages(zone);
 172                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
 173                         n += saveable(zone, &zone_pfn);
 174         }
 175         return n;
 176 }
 177
 178 static void copy_data_pages(struct pbe *pblist)
 179 {
 180         struct zone *zone;
 181         unsigned long zone_pfn;
 182         struct pbe *pbe, *p;
 183
 184         pbe = pblist;
 185         for_each_zone (zone) {
 186                 if (is_highmem(zone))
 187                         continue;
 188                 mark_free_pages(zone);
 189                 /* This is necessary for swsusp_free() */
 190                 for_each_pb_page (p, pblist)
 191                         SetPageNosaveFree(virt_to_page(p));
 192                 for_each_pbe (p, pblist)
 193                         SetPageNosaveFree(virt_to_page(p->address));
 194                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) {
 195                         if (saveable(zone, &zone_pfn)) {
 196                                 struct page *page;
 197                                 page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
 198                                 BUG_ON(!pbe);
 199                                 pbe->orig_address = (unsigned long)page_address(page);
 200                                 /* copy_page is not usable for copying task structs. */
 201                                 memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE);
 202                                 pbe = pbe->next;
 203                         }
 204                 }
 205         }
 206         BUG_ON(pbe);
 207 }
 208
 209
 210 /**
 211  *      free_pagedir - free pages allocated with alloc_pagedir()
 212  */
 213
 214 void free_pagedir(struct pbe *pblist)
 215 {
 216         struct pbe *pbe;
 217
 218         while (pblist) {
 219                 pbe = (pblist + PB_PAGE_SKIP)->next;
 220                 ClearPageNosave(virt_to_page(pblist));
 221                 ClearPageNosaveFree(virt_to_page(pblist));
 222                 free_page((unsigned long)pblist);
 223                 pblist = pbe;
 224         }
 225 }
 226
 227 /**
 228  *      fill_pb_page - Create a list of PBEs on a given memory page
 229  */
 230
 231 static inline void fill_pb_page(struct pbe *pbpage)
 232 {
 233         struct pbe *p;
 234
 235         p = pbpage;
 236         pbpage += PB_PAGE_SKIP;
 237         do
 238                 p->next = p + 1;
 239         while (++p < pbpage);
 240 }
 241
 242 /**
 243  *      create_pbe_list - Create a list of PBEs on top of a given chain
 244  *      of memory pages allocated with alloc_pagedir()
 245  */
 246
 247 void create_pbe_list(struct pbe *pblist, unsigned int nr_pages)
 248 {
 249         struct pbe *pbpage, *p;
 250         unsigned int num = PBES_PER_PAGE;
 251
 252         for_each_pb_page (pbpage, pblist) {
 253                 if (num >= nr_pages)
 254                         break;
 255
 256                 fill_pb_page(pbpage);
 257                 num += PBES_PER_PAGE;
 258         }
 259         if (pbpage) {
 260                 for (num -= PBES_PER_PAGE - 1, p = pbpage; num < nr_pages; p++, num++)
 261                         p->next = p + 1;
 262                 p->next = NULL;
 263         }
 264         pr_debug("create_pbe_list(): initialized %d PBEs\n", num);
 265 }
 266
 267 /**
 268  *      @safe_needed - on resume, for storing the PBE list and the image,
 269  *      we can only use memory pages that do not conflict with the pages
 270  *      which had been used before suspend.
 271  *
 272  *      The unsafe pages are marked with the PG_nosave_free flag
 273  *
 274  *      Allocated but unusable (ie eaten) memory pages should be marked
 275  *      so that swsusp_free() can release them
 276  */
 277
 278 static inline void *alloc_image_page(gfp_t gfp_mask, int safe_needed)
 279 {
 280         void *res;
 281
 282         if (safe_needed)
 283                 do {
 284                         res = (void *)get_zeroed_page(gfp_mask);
 285                         if (res && PageNosaveFree(virt_to_page(res)))
 286                                 /* This is for swsusp_free() */
 287                                 SetPageNosave(virt_to_page(res));
 288                 } while (res && PageNosaveFree(virt_to_page(res)));
 289         else
 290                 res = (void *)get_zeroed_page(gfp_mask);
 291         if (res) {
 292                 SetPageNosave(virt_to_page(res));
 293                 SetPageNosaveFree(virt_to_page(res));
 294         }
 295         return res;
 296 }
 297
 298 unsigned long get_safe_page(gfp_t gfp_mask)
 299 {
 300         return (unsigned long)alloc_image_page(gfp_mask, 1);
 301 }
 302
 303 /**
 304  *      alloc_pagedir - Allocate the page directory.
 305  *
 306  *      First, determine exactly how many pages we need and
 307  *      allocate them.
 308  *
 309  *      We arrange the pages in a chain: each page is an array of PBES_PER_PAGE
 310  *      struct pbe elements (pbes) and the last element in the page points
 311  *      to the next page.
 312  *
 313  *      On each page we set up a list of struct_pbe elements.
 314  */
 315
 316 struct pbe *alloc_pagedir(unsigned int nr_pages, gfp_t gfp_mask, int safe_needed)
 317 {
 318         unsigned int num;
 319         struct pbe *pblist, *pbe;
 320
 321         if (!nr_pages)
 322                 return NULL;
 323
 324         pr_debug("alloc_pagedir(): nr_pages = %d\n", nr_pages);
 325         pblist = alloc_image_page(gfp_mask, safe_needed);
 326         /* FIXME: rewrite this ugly loop */
 327         for (pbe = pblist, num = PBES_PER_PAGE; pbe && num < nr_pages;
 328                         pbe = pbe->next, num += PBES_PER_PAGE) {
 329                 pbe += PB_PAGE_SKIP;
 330                 pbe->next = alloc_image_page(gfp_mask, safe_needed);
 331         }
 332         if (!pbe) { /* get_zeroed_page() failed */
 333                 free_pagedir(pblist);
 334                 pblist = NULL;
 335         }
 336         return pblist;
 337 }
 338
 339 /**
 340  * Free pages we allocated for suspend. Suspend pages are alocated
 341  * before atomic copy, so we need to free them after resume.
 342  */
 343
 344 void swsusp_free(void)
 345 {
 346         struct zone *zone;
 347         unsigned long zone_pfn;
 348
 349         for_each_zone(zone) {
 350                 for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn)
 351                         if (pfn_valid(zone_pfn + zone->zone_start_pfn)) {
 352                                 struct page *page;
 353                                 page = pfn_to_page(zone_pfn + zone->zone_start_pfn);
 354                                 if (PageNosave(page) && PageNosaveFree(page)) {
 355                                         ClearPageNosave(page);
 356                                         ClearPageNosaveFree(page);
 357                                         free_page((long) page_address(page));
 358                                 }
 359                         }
 360         }
 361 }
 362
 363
 364 /**
 365  *      enough_free_mem - Make sure we enough free memory to snapshot.
 366  *
 367  *      Returns TRUE or FALSE after checking the number of available
 368  *      free pages.
 369  */
 370
 371 static int enough_free_mem(unsigned int nr_pages)
 372 {
 373         pr_debug("swsusp: available memory: %u pages\n", nr_free_pages());
 374         return nr_free_pages() > (nr_pages + PAGES_FOR_IO +
 375                 (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE);
 376 }
 377
 378 int alloc_data_pages(struct pbe *pblist, gfp_t gfp_mask, int safe_needed)
 379 {
 380         struct pbe *p;
 381
 382         for_each_pbe (p, pblist) {
 383                 p->address = (unsigned long)alloc_image_page(gfp_mask, safe_needed);
 384                 if (!p->address)
 385                         return -ENOMEM;
 386         }
 387         return 0;
 388 }
 389
 390 static struct pbe *swsusp_alloc(unsigned int nr_pages)
 391 {
 392         struct pbe *pblist;
 393
 394         if (!(pblist = alloc_pagedir(nr_pages, GFP_ATOMIC | __GFP_COLD, 0))) {
 395                 printk(KERN_ERR "suspend: Allocating pagedir failed.\n");
 396                 return NULL;
 397         }
 398         create_pbe_list(pblist, nr_pages);
 399
 400         if (alloc_data_pages(pblist, GFP_ATOMIC | __GFP_COLD, 0)) {
 401                 printk(KERN_ERR "suspend: Allocating image pages failed.\n");
 402                 swsusp_free();
 403                 return NULL;
 404         }
 405
 406         return pblist;
 407 }
 408
 409 asmlinkage int swsusp_save(void)
 410 {
 411         unsigned int nr_pages;
 412
 413         pr_debug("swsusp: critical section: \n");
 414
 415         drain_local_pages();
 416         nr_pages = count_data_pages();
 417         printk("swsusp: Need to copy %u pages\n", nr_pages);
 418
 419         pr_debug("swsusp: pages needed: %u + %lu + %u, free: %u\n",
 420                  nr_pages,
 421                  (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE,
 422                  PAGES_FOR_IO, nr_free_pages());
 423
 424         /* This is needed because of the fixed size of swsusp_info */
 425         if (MAX_PBES < (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE)
 426                 return -ENOSPC;
 427
 428         if (!enough_free_mem(nr_pages)) {
 429                 printk(KERN_ERR "swsusp: Not enough free memory\n");
 430                 return -ENOMEM;
 431         }
 432
 433         pagedir_nosave = swsusp_alloc(nr_pages);
 434         if (!pagedir_nosave)
 435                 return -ENOMEM;
 436
 437         /* During allocating of suspend pagedir, new cold pages may appear.
 438          * Kill them.
 439          */
 440         drain_local_pages();
 441         copy_data_pages(pagedir_nosave);
 442
 443         /*
 444          * End of critical section. From now on, we can write to memory,
 445          * but we should not touch disk. This specially means we must _not_
 446          * touch swap space! Except we must write out our image of course.
 447          */
 448
 449         nr_copy_pages = nr_pages;
 450
 451         printk("swsusp: critical section/: done (%d pages copied)\n", nr_pages);
 452         return 0;
 453 }