arch/x86/mm/ioremap.c

   1 /*
   2  * Re-map IO memory to kernel address space so that we can access it.
   3  * This is needed for high PCI addresses that aren't mapped in the
   4  * 640k-1MB IO memory area on PC's
   5  *
   6  * (C) Copyright 1995 1996 Linus Torvalds
   7  */
   8
   9 #include <linux/bootmem.h>
  10 #include <linux/init.h>
  11 #include <linux/io.h>
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include <linux/vmalloc.h>
  15
  16 #include <asm/cacheflush.h>
  17 #include <asm/e820.h>
  18 #include <asm/fixmap.h>
  19 #include <asm/pgtable.h>
  20 #include <asm/tlbflush.h>
  21 #include <asm/pgalloc.h>
  22
  23 enum ioremap_mode {
  24         IOR_MODE_UNCACHED,
  25         IOR_MODE_CACHED,
  26 };
  27
  28 #ifdef CONFIG_X86_64
  29
  30 unsigned long __phys_addr(unsigned long x)
  31 {
  32         if (x >= __START_KERNEL_map)
  33                 return x - __START_KERNEL_map + phys_base;
  34         return x - PAGE_OFFSET;
  35 }
  36 EXPORT_SYMBOL(__phys_addr);
  37
  38 #endif
  39
  40 int page_is_ram(unsigned long pagenr)
  41 {
  42         unsigned long addr, end;
  43         int i;
  44
  45         /*
  46          * A special case is the first 4Kb of memory;
  47          * This is a BIOS owned area, not kernel ram, but generally
  48          * not listed as such in the E820 table.
  49          */
  50         if (pagenr == 0)
  51                 return 0;
  52
  53         /*
  54          * Second special case: Some BIOSen report the PC BIOS
  55          * area (640->1Mb) as ram even though it is not.
  56          */
  57         if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
  58                     pagenr < (BIOS_END >> PAGE_SHIFT))
  59                 return 0;
  60
  61         for (i = 0; i < e820.nr_map; i++) {
  62                 /*
  63                  * Not usable memory:
  64                  */
  65                 if (e820.map[i].type != E820_RAM)
  66                         continue;
  67                 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
  68                 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
  69
  70
  71                 if ((pagenr >= addr) && (pagenr < end))
  72                         return 1;
  73         }
  74         return 0;
  75 }
  76
  77 /*
  78  * Fix up the linear direct mapping of the kernel to avoid cache attribute
  79  * conflicts.
  80  */
  81 static int ioremap_change_attr(unsigned long vaddr, unsigned long size,
  82                                enum ioremap_mode mode)
  83 {
  84         unsigned long nrpages = size >> PAGE_SHIFT;
  85         int err;
  86
  87         switch (mode) {
  88         case IOR_MODE_UNCACHED:
  89         default:
  90                 err = set_memory_uc(vaddr, nrpages);
  91                 break;
  92         case IOR_MODE_CACHED:
  93                 err = set_memory_wb(vaddr, nrpages);
  94                 break;
  95         }
  96
  97         return err;
  98 }
  99
 100 /*
 101  * Remap an arbitrary physical address space into the kernel virtual
 102  * address space. Needed when the kernel wants to access high addresses
 103  * directly.
 104  *
 105  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 106  * have to convert them into an offset in a page-aligned mapping, but the
 107  * caller shouldn't need to know that small detail.
 108  */
 109 static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
 110                                enum ioremap_mode mode)
 111 {
 112         unsigned long pfn, offset, last_addr, vaddr;
 113         struct vm_struct *area;
 114         pgprot_t prot;
 115
 116         /* Don't allow wraparound or zero size */
 117         last_addr = phys_addr + size - 1;
 118         if (!size || last_addr < phys_addr)
 119                 return NULL;
 120
 121         /*
 122          * Don't remap the low PCI/ISA area, it's always mapped..
 123          */
 124         if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
 125                 return (__force void __iomem *)phys_to_virt(phys_addr);
 126
 127         /*
 128          * Don't allow anybody to remap normal RAM that we're using..
 129          */
 130         for (pfn = phys_addr >> PAGE_SHIFT; pfn < max_pfn_mapped &&
 131              (pfn << PAGE_SHIFT) < last_addr; pfn++) {
 132                 if (page_is_ram(pfn) && pfn_valid(pfn) &&
 133                     !PageReserved(pfn_to_page(pfn)))
 134                         return NULL;
 135         }
 136
 137         switch (mode) {
 138         case IOR_MODE_UNCACHED:
 139         default:
 140                 prot = PAGE_KERNEL_NOCACHE;
 141                 break;
 142         case IOR_MODE_CACHED:
 143                 prot = PAGE_KERNEL;
 144                 break;
 145         }
 146
 147         /*
 148          * Mappings have to be page-aligned
 149          */
 150         offset = phys_addr & ~PAGE_MASK;
 151         phys_addr &= PAGE_MASK;
 152         size = PAGE_ALIGN(last_addr+1) - phys_addr;
 153
 154         /*
 155          * Ok, go for it..
 156          */
 157         area = get_vm_area(size, VM_IOREMAP);
 158         if (!area)
 159                 return NULL;
 160         area->phys_addr = phys_addr;
 161         vaddr = (unsigned long) area->addr;
 162         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
 163                 remove_vm_area((void *)(vaddr & PAGE_MASK));
 164                 return NULL;
 165         }
 166
 167         if (ioremap_change_attr(vaddr, size, mode) < 0) {
 168                 vunmap(area->addr);
 169                 return NULL;
 170         }
 171
 172         return (void __iomem *) (vaddr + offset);
 173 }
 174
 175 /**
 176  * ioremap_nocache     -   map bus memory into CPU space
 177  * @offset:    bus address of the memory
 178  * @size:      size of the resource to map
 179  *
 180  * ioremap_nocache performs a platform specific sequence of operations to
 181  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 182  * writew/writel functions and the other mmio helpers. The returned
 183  * address is not guaranteed to be usable directly as a virtual
 184  * address.
 185  *
 186  * This version of ioremap ensures that the memory is marked uncachable
 187  * on the CPU as well as honouring existing caching rules from things like
 188  * the PCI bus. Note that there are other caches and buffers on many
 189  * busses. In particular driver authors should read up on PCI writes
 190  *
 191  * It's useful if some control registers are in such an area and
 192  * write combining or read caching is not desirable:
 193  *
 194  * Must be freed with iounmap.
 195  */
 196 void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
 197 {
 198         return __ioremap(phys_addr, size, IOR_MODE_UNCACHED);
 199 }
 200 EXPORT_SYMBOL(ioremap_nocache);
 201
 202 void __iomem *ioremap_cache(unsigned long phys_addr, unsigned long size)
 203 {
 204         return __ioremap(phys_addr, size, IOR_MODE_CACHED);
 205 }
 206 EXPORT_SYMBOL(ioremap_cache);
 207
 208 /**
 209  * iounmap - Free a IO remapping
 210  * @addr: virtual address from ioremap_*
 211  *
 212  * Caller must ensure there is only one unmapping for the same pointer.
 213  */
 214 void iounmap(volatile void __iomem *addr)
 215 {
 216         struct vm_struct *p, *o;
 217
 218         if ((void __force *)addr <= high_memory)
 219                 return;
 220
 221         /*
 222          * __ioremap special-cases the PCI/ISA range by not instantiating a
 223          * vm_area and by simply returning an address into the kernel mapping
 224          * of ISA space.   So handle that here.
 225          */
 226         if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
 227             addr < phys_to_virt(ISA_END_ADDRESS))
 228                 return;
 229
 230         addr = (volatile void __iomem *)
 231                 (PAGE_MASK & (unsigned long __force)addr);
 232
 233         /* Use the vm area unlocked, assuming the caller
 234            ensures there isn't another iounmap for the same address
 235            in parallel. Reuse of the virtual address is prevented by
 236            leaving it in the global lists until we're done with it.
 237            cpa takes care of the direct mappings. */
 238         read_lock(&vmlist_lock);
 239         for (p = vmlist; p; p = p->next) {
 240                 if (p->addr == addr)
 241                         break;
 242         }
 243         read_unlock(&vmlist_lock);
 244
 245         if (!p) {
 246                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
 247                 dump_stack();
 248                 return;
 249         }
 250
 251         /* Finally remove it */
 252         o = remove_vm_area((void *)addr);
 253         BUG_ON(p != o || o == NULL);
 254         kfree(p);
 255 }
 256 EXPORT_SYMBOL(iounmap);
 257
 258 #ifdef CONFIG_X86_32
 259
 260 int __initdata early_ioremap_debug;
 261
 262 static int __init early_ioremap_debug_setup(char *str)
 263 {
 264         early_ioremap_debug = 1;
 265
 266         return 0;
 267 }
 268 early_param("early_ioremap_debug", early_ioremap_debug_setup);
 269
 270 static __initdata int after_paging_init;
 271 static __initdata pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
 272                                 __attribute__((aligned(PAGE_SIZE)));
 273
 274 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
 275 {
 276         /* Don't assume we're using swapper_pg_dir at this point */
 277         pgd_t *base = __va(read_cr3());
 278         pgd_t *pgd = &base[pgd_index(addr)];
 279         pud_t *pud = pud_offset(pgd, addr);
 280         pmd_t *pmd = pmd_offset(pud, addr);
 281
 282         return pmd;
 283 }
 284
 285 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
 286 {
 287         return &bm_pte[pte_index(addr)];
 288 }
 289
 290 void __init early_ioremap_init(void)
 291 {
 292         pmd_t *pmd;
 293
 294         if (early_ioremap_debug)
 295                 printk(KERN_INFO "early_ioremap_init()\n");
 296
 297         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 298         memset(bm_pte, 0, sizeof(bm_pte));
 299         pmd_populate_kernel(&init_mm, pmd, bm_pte);
 300
 301         /*
 302          * The boot-ioremap range spans multiple pmds, for which
 303          * we are not prepared:
 304          */
 305         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
 306                 WARN_ON(1);
 307                 printk(KERN_WARNING "pmd %p != %p\n",
 308                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
 309                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 310                         fix_to_virt(FIX_BTMAP_BEGIN));
 311                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
 312                         fix_to_virt(FIX_BTMAP_END));
 313
 314                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
 315                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
 316                        FIX_BTMAP_BEGIN);
 317         }
 318 }
 319
 320 void __init early_ioremap_clear(void)
 321 {
 322         pmd_t *pmd;
 323
 324         if (early_ioremap_debug)
 325                 printk(KERN_INFO "early_ioremap_clear()\n");
 326
 327         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 328         pmd_clear(pmd);
 329         paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
 330         __flush_tlb_all();
 331 }
 332
 333 void __init early_ioremap_reset(void)
 334 {
 335         enum fixed_addresses idx;
 336         unsigned long addr, phys;
 337         pte_t *pte;
 338
 339         after_paging_init = 1;
 340         for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
 341                 addr = fix_to_virt(idx);
 342                 pte = early_ioremap_pte(addr);
 343                 if (pte_present(*pte)) {
 344                         phys = pte_val(*pte) & PAGE_MASK;
 345                         set_fixmap(idx, phys);
 346                 }
 347         }
 348 }
 349
 350 static void __init __early_set_fixmap(enum fixed_addresses idx,
 351                                    unsigned long phys, pgprot_t flags)
 352 {
 353         unsigned long addr = __fix_to_virt(idx);
 354         pte_t *pte;
 355
 356         if (idx >= __end_of_fixed_addresses) {
 357                 BUG();
 358                 return;
 359         }
 360         pte = early_ioremap_pte(addr);
 361         if (pgprot_val(flags))
 362                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
 363         else
 364                 pte_clear(NULL, addr, pte);
 365         __flush_tlb_one(addr);
 366 }
 367
 368 static inline void __init early_set_fixmap(enum fixed_addresses idx,
 369                                         unsigned long phys)
 370 {
 371         if (after_paging_init)
 372                 set_fixmap(idx, phys);
 373         else
 374                 __early_set_fixmap(idx, phys, PAGE_KERNEL);
 375 }
 376
 377 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
 378 {
 379         if (after_paging_init)
 380                 clear_fixmap(idx);
 381         else
 382                 __early_set_fixmap(idx, 0, __pgprot(0));
 383 }
 384
 385
 386 int __initdata early_ioremap_nested;
 387
 388 static int __init check_early_ioremap_leak(void)
 389 {
 390         if (!early_ioremap_nested)
 391                 return 0;
 392
 393         printk(KERN_WARNING
 394                "Debug warning: early ioremap leak of %d areas detected.\n",
 395                early_ioremap_nested);
 396         printk(KERN_WARNING
 397                "please boot with early_ioremap_debug and report the dmesg.\n");
 398         WARN_ON(1);
 399
 400         return 1;
 401 }
 402 late_initcall(check_early_ioremap_leak);
 403
 404 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
 405 {
 406         unsigned long offset, last_addr;
 407         unsigned int nrpages, nesting;
 408         enum fixed_addresses idx0, idx;
 409
 410         WARN_ON(system_state != SYSTEM_BOOTING);
 411
 412         nesting = early_ioremap_nested;
 413         if (early_ioremap_debug) {
 414                 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
 415                        phys_addr, size, nesting);
 416                 dump_stack();
 417         }
 418
 419         /* Don't allow wraparound or zero size */
 420         last_addr = phys_addr + size - 1;
 421         if (!size || last_addr < phys_addr) {
 422                 WARN_ON(1);
 423                 return NULL;
 424         }
 425
 426         if (nesting >= FIX_BTMAPS_NESTING) {
 427                 WARN_ON(1);
 428                 return NULL;
 429         }
 430         early_ioremap_nested++;
 431         /*
 432          * Mappings have to be page-aligned
 433          */
 434         offset = phys_addr & ~PAGE_MASK;
 435         phys_addr &= PAGE_MASK;
 436         size = PAGE_ALIGN(last_addr) - phys_addr;
 437
 438         /*
 439          * Mappings have to fit in the FIX_BTMAP area.
 440          */
 441         nrpages = size >> PAGE_SHIFT;
 442         if (nrpages > NR_FIX_BTMAPS) {
 443                 WARN_ON(1);
 444                 return NULL;
 445         }
 446
 447         /*
 448          * Ok, go for it..
 449          */
 450         idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 451         idx = idx0;
 452         while (nrpages > 0) {
 453                 early_set_fixmap(idx, phys_addr);
 454                 phys_addr += PAGE_SIZE;
 455                 --idx;
 456                 --nrpages;
 457         }
 458         if (early_ioremap_debug)
 459                 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
 460
 461         return (void *) (offset + fix_to_virt(idx0));
 462 }
 463
 464 void __init early_iounmap(void *addr, unsigned long size)
 465 {
 466         unsigned long virt_addr;
 467         unsigned long offset;
 468         unsigned int nrpages;
 469         enum fixed_addresses idx;
 470         unsigned int nesting;
 471
 472         nesting = --early_ioremap_nested;
 473         WARN_ON(nesting < 0);
 474
 475         if (early_ioremap_debug) {
 476                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
 477                        size, nesting);
 478                 dump_stack();
 479         }
 480
 481         virt_addr = (unsigned long)addr;
 482         if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
 483                 WARN_ON(1);
 484                 return;
 485         }
 486         offset = virt_addr & ~PAGE_MASK;
 487         nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
 488
 489         idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 490         while (nrpages > 0) {
 491                 early_clear_fixmap(idx);
 492                 --idx;
 493                 --nrpages;
 494         }
 495 }
 496
 497 void __this_fixmap_does_not_exist(void)
 498 {
 499         WARN_ON(1);
 500 }
 501
 502 #endif /* CONFIG_X86_32 */