projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
PCI: fix section mismatch on update_res()
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / pci / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
5e0d2a6f 35#include <linux/timer.h>
38717946 36#include <linux/iova.h>
5d450806 37#include <linux/iommu.h>
38717946 38#include <linux/intel-iommu.h>
f59c7b69 39#include <linux/sysdev.h>
69575d38 40#include <linux/tboot.h>
adb2fe02 41#include <linux/dmi.h>
ba395927 42#include <asm/cacheflush.h>
46a7fa27 43#include <asm/iommu.h>
ba395927
KA		 44#include "pci.h"
45
5b6985ce
FY		 46#define ROOT_SIZE VTD_PAGE_SIZE
47#define CONTEXT_SIZE VTD_PAGE_SIZE
48
ba395927
KA		 49#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 51#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 52
53#define IOAPIC_RANGE_START (0xfee00000)
54#define IOAPIC_RANGE_END (0xfeefffff)
55#define IOVA_START_ADDR (0x1000)
56
57#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
4ed0d3e6
FY		 59#define MAX_AGAW_WIDTH 64
60
2ebe3151
DW		 61#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 69
f27be03b 70#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 71#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 72#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 73
fd18de50 74
dd4e8319
DW		 75/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
76 are never going to work. */
77static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
78{
79 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
80}
81
82static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
83{
84 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
85}
86static inline unsigned long page_to_dma_pfn(struct page *pg)
87{
88 return mm_to_dma_pfn(page_to_pfn(pg));
89}
90static inline unsigned long virt_to_dma_pfn(void *p)
91{
92 return page_to_dma_pfn(virt_to_page(p));
93}
94
d9630fe9
WH		 95/* global iommu list, set NULL for ignored DMAR units */
96static struct intel_iommu **g_iommus;
97
e0fc7e0b 98static void __init check_tylersburg_isoch(void);
9af88143
DW		 99static int rwbf_quirk;
100
46b08e1a
MM		 101/*
102 * 0: Present
103 * 1-11: Reserved
104 * 12-63: Context Ptr (12 - (haw-1))
105 * 64-127: Reserved
106 */
107struct root_entry {
108 u64 val;
109 u64 rsvd1;
110};
111#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
112static inline bool root_present(struct root_entry *root)
113{
114 return (root->val & 1);
115}
116static inline void set_root_present(struct root_entry *root)
117{
118 root->val |= 1;
119}
120static inline void set_root_value(struct root_entry *root, unsigned long value)
121{
122 root->val |= value & VTD_PAGE_MASK;
123}
124
125static inline struct context_entry *
126get_context_addr_from_root(struct root_entry *root)
127{
128 return (struct context_entry *)
129 (root_present(root)?phys_to_virt(
130 root->val & VTD_PAGE_MASK) :
131 NULL);
132}
133
7a8fc25e
MM		 134/*
135 * low 64 bits:
136 * 0: present
137 * 1: fault processing disable
138 * 2-3: translation type
139 * 12-63: address space root
140 * high 64 bits:
141 * 0-2: address width
142 * 3-6: aval
143 * 8-23: domain id
144 */
145struct context_entry {
146 u64 lo;
147 u64 hi;
148};
c07e7d21
MM		 149
150static inline bool context_present(struct context_entry *context)
151{
152 return (context->lo & 1);
153}
154static inline void context_set_present(struct context_entry *context)
155{
156 context->lo |= 1;
157}
158
159static inline void context_set_fault_enable(struct context_entry *context)
160{
161 context->lo &= (((u64)-1) << 2) | 1;
162}
163
c07e7d21
MM		 164static inline void context_set_translation_type(struct context_entry *context,
165 unsigned long value)
166{
167 context->lo &= (((u64)-1) << 4) | 3;
168 context->lo |= (value & 3) << 2;
169}
170
171static inline void context_set_address_root(struct context_entry *context,
172 unsigned long value)
173{
174 context->lo |= value & VTD_PAGE_MASK;
175}
176
177static inline void context_set_address_width(struct context_entry *context,
178 unsigned long value)
179{
180 context->hi |= value & 7;
181}
182
183static inline void context_set_domain_id(struct context_entry *context,
184 unsigned long value)
185{
186 context->hi |= (value & ((1 << 16) - 1)) << 8;
187}
188
189static inline void context_clear_entry(struct context_entry *context)
190{
191 context->lo = 0;
192 context->hi = 0;
193}
7a8fc25e 194
622ba12a
MM		 195/*
196 * 0: readable
197 * 1: writable
198 * 2-6: reserved
199 * 7: super page
9cf06697
SY		 200 * 8-10: available
201 * 11: snoop behavior
622ba12a
MM		 202 * 12-63: Host physcial address
203 */
204struct dma_pte {
205 u64 val;
206};
622ba12a 207
19c239ce
MM		 208static inline void dma_clear_pte(struct dma_pte *pte)
209{
210 pte->val = 0;
211}
212
213static inline void dma_set_pte_readable(struct dma_pte *pte)
214{
215 pte->val |= DMA_PTE_READ;
216}
217
218static inline void dma_set_pte_writable(struct dma_pte *pte)
219{
220 pte->val |= DMA_PTE_WRITE;
221}
222
9cf06697
SY		 223static inline void dma_set_pte_snp(struct dma_pte *pte)
224{
225 pte->val |= DMA_PTE_SNP;
226}
227
19c239ce
MM		 228static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
229{
230 pte->val = (pte->val & ~3) | (prot & 3);
231}
232
233static inline u64 dma_pte_addr(struct dma_pte *pte)
234{
c85994e4
DW		 235#ifdef CONFIG_64BIT
236 return pte->val & VTD_PAGE_MASK;
237#else
238 /* Must have a full atomic 64-bit read */
239 return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
240#endif
19c239ce
MM		 241}
242
dd4e8319 243static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 244{
dd4e8319 245 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 246}
247
248static inline bool dma_pte_present(struct dma_pte *pte)
249{
250 return (pte->val & 3) != 0;
251}
622ba12a 252
75e6bf96
DW		 253static inline int first_pte_in_page(struct dma_pte *pte)
254{
255 return !((unsigned long)pte & ~VTD_PAGE_MASK);
256}
257
2c2e2c38
FY		 258/*
259 * This domain is a statically identity mapping domain.
260 * 1. This domain creats a static 1:1 mapping to all usable memory.
261 * 2. It maps to each iommu if successful.
262 * 3. Each iommu mapps to this domain if successful.
263 */
19943b0e
DW		 264static struct dmar_domain *si_domain;
265static int hw_pass_through = 1;
2c2e2c38 266
3b5410e7 267/* devices under the same p2p bridge are owned in one domain */
cdc7b837 268#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 269
1ce28feb
WH		 270/* domain represents a virtual machine, more than one devices
271 * across iommus may be owned in one domain, e.g. kvm guest.
272 */
273#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
274
2c2e2c38
FY		 275/* si_domain contains mulitple devices */
276#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
277
99126f7c
MM		 278struct dmar_domain {
279 int id; /* domain id */
8c11e798 280 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
99126f7c
MM		 281
282 struct list_head devices; /* all devices' list */
283 struct iova_domain iovad; /* iova's that belong to this domain */
284
285 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 286 int gaw; /* max guest address width */
287
288 /* adjusted guest address width, 0 is level 2 30-bit */
289 int agaw;
290
3b5410e7 291 int flags; /* flags to find out type of domain */
8e604097
WH		 292
293 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 294 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d
WH		 295 int iommu_count; /* reference count of iommu */
296 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 297 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 298};
299
a647dacb
MM		 300/* PCI domain-device relationship */
301struct device_domain_info {
302 struct list_head link; /* link to domain siblings */
303 struct list_head global; /* link to global list */
276dbf99
DW		 304 int segment; /* PCI domain */
305 u8 bus; /* PCI bus number */
a647dacb
MM		 306 u8 devfn; /* PCI devfn number */
307 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
93a23a72 308 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 309 struct dmar_domain *domain; /* pointer to domain */
310};
311
5e0d2a6f 312static void flush_unmaps_timeout(unsigned long data);
313
314DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
315
80b20dd8 316#define HIGH_WATER_MARK 250
317struct deferred_flush_tables {
318 int next;
319 struct iova *iova[HIGH_WATER_MARK];
320 struct dmar_domain *domain[HIGH_WATER_MARK];
321};
322
323static struct deferred_flush_tables *deferred_flush;
324
5e0d2a6f 325/* bitmap for indexing intel_iommus */
5e0d2a6f 326static int g_num_of_iommus;
327
328static DEFINE_SPINLOCK(async_umap_flush_lock);
329static LIST_HEAD(unmaps_to_do);
330
331static int timer_on;
332static long list_size;
5e0d2a6f 333
ba395927
KA		 334static void domain_remove_dev_info(struct dmar_domain *domain);
335
0cd5c3c8
KM		 336#ifdef CONFIG_DMAR_DEFAULT_ON
337int dmar_disabled = 0;
338#else
339int dmar_disabled = 1;
340#endif /*CONFIG_DMAR_DEFAULT_ON*/
341
ba395927 342static int __initdata dmar_map_gfx = 1;
7d3b03ce 343static int dmar_forcedac;
5e0d2a6f 344static int intel_iommu_strict;
ba395927
KA		 345
346#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
347static DEFINE_SPINLOCK(device_domain_lock);
348static LIST_HEAD(device_domain_list);
349
a8bcbb0d
JR		 350static struct iommu_ops intel_iommu_ops;
351
ba395927
KA		 352static int __init intel_iommu_setup(char *str)
353{
354 if (!str)
355 return -EINVAL;
356 while (*str) {
0cd5c3c8
KM		 357 if (!strncmp(str, "on", 2)) {
358 dmar_disabled = 0;
359 printk(KERN_INFO "Intel-IOMMU: enabled\n");
360 } else if (!strncmp(str, "off", 3)) {
ba395927 361 dmar_disabled = 1;
0cd5c3c8 362 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 363 } else if (!strncmp(str, "igfx_off", 8)) {
364 dmar_map_gfx = 0;
365 printk(KERN_INFO
366 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 367 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 368 printk(KERN_INFO
7d3b03ce
KA		 369 "Intel-IOMMU: Forcing DAC for PCI devices\n");
370 dmar_forcedac = 1;
5e0d2a6f 371 } else if (!strncmp(str, "strict", 6)) {
372 printk(KERN_INFO
373 "Intel-IOMMU: disable batched IOTLB flush\n");
374 intel_iommu_strict = 1;
ba395927
KA		 375 }
376
377 str += strcspn(str, ",");
378 while (*str == ',')
379 str++;
380 }
381 return 0;
382}
383__setup("intel_iommu=", intel_iommu_setup);
384
385static struct kmem_cache *iommu_domain_cache;
386static struct kmem_cache *iommu_devinfo_cache;
387static struct kmem_cache *iommu_iova_cache;
388
eb3fa7cb
KA		 389static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
390{
391 unsigned int flags;
392 void *vaddr;
393
394 /* trying to avoid low memory issues */
395 flags = current->flags & PF_MEMALLOC;
396 current->flags |= PF_MEMALLOC;
397 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
398 current->flags &= (~PF_MEMALLOC | flags);
399 return vaddr;
400}
401
402
ba395927
KA		 403static inline void *alloc_pgtable_page(void)
404{
eb3fa7cb
KA		 405 unsigned int flags;
406 void *vaddr;
407
408 /* trying to avoid low memory issues */
409 flags = current->flags & PF_MEMALLOC;
410 current->flags |= PF_MEMALLOC;
411 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
412 current->flags &= (~PF_MEMALLOC | flags);
413 return vaddr;
ba395927
KA		 414}
415
416static inline void free_pgtable_page(void *vaddr)
417{
418 free_page((unsigned long)vaddr);
419}
420
421static inline void *alloc_domain_mem(void)
422{
eb3fa7cb 423 return iommu_kmem_cache_alloc(iommu_domain_cache);
ba395927
KA		 424}
425
38717946 426static void free_domain_mem(void *vaddr)
ba395927
KA		 427{
428 kmem_cache_free(iommu_domain_cache, vaddr);
429}
430
431static inline void * alloc_devinfo_mem(void)
432{
eb3fa7cb 433 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
ba395927
KA		 434}
435
436static inline void free_devinfo_mem(void *vaddr)
437{
438 kmem_cache_free(iommu_devinfo_cache, vaddr);
439}
440
441struct iova *alloc_iova_mem(void)
442{
eb3fa7cb 443 return iommu_kmem_cache_alloc(iommu_iova_cache);
ba395927
KA		 444}
445
446void free_iova_mem(struct iova *iova)
447{
448 kmem_cache_free(iommu_iova_cache, iova);
449}
450
1b573683
WH		 451
452static inline int width_to_agaw(int width);
453
4ed0d3e6 454static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 455{
456 unsigned long sagaw;
457 int agaw = -1;
458
459 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 460 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 461 agaw >= 0; agaw--) {
462 if (test_bit(agaw, &sagaw))
463 break;
464 }
465
466 return agaw;
467}
468
4ed0d3e6
FY		 469/*
470 * Calculate max SAGAW for each iommu.
471 */
472int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
473{
474 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
475}
476
477/*
478 * calculate agaw for each iommu.
479 * "SAGAW" may be different across iommus, use a default agaw, and
480 * get a supported less agaw for iommus that don't support the default agaw.
481 */
482int iommu_calculate_agaw(struct intel_iommu *iommu)
483{
484 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
485}
486
2c2e2c38 487/* This functionin only returns single iommu in a domain */
8c11e798
WH		 488static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
489{
490 int iommu_id;
491
2c2e2c38 492 /* si_domain and vm domain should not get here. */
1ce28feb 493 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 494 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 495
8c11e798
WH		 496 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
497 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
498 return NULL;
499
500 return g_iommus[iommu_id];
501}
502
8e604097
WH		 503static void domain_update_iommu_coherency(struct dmar_domain *domain)
504{
505 int i;
506
507 domain->iommu_coherency = 1;
508
509 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
510 for (; i < g_num_of_iommus; ) {
511 if (!ecap_coherent(g_iommus[i]->ecap)) {
512 domain->iommu_coherency = 0;
513 break;
514 }
515 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
516 }
517}
518
58c610bd
SY		 519static void domain_update_iommu_snooping(struct dmar_domain *domain)
520{
521 int i;
522
523 domain->iommu_snooping = 1;
524
525 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
526 for (; i < g_num_of_iommus; ) {
527 if (!ecap_sc_support(g_iommus[i]->ecap)) {
528 domain->iommu_snooping = 0;
529 break;
530 }
531 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
532 }
533}
534
535/* Some capabilities may be different across iommus */
536static void domain_update_iommu_cap(struct dmar_domain *domain)
537{
538 domain_update_iommu_coherency(domain);
539 domain_update_iommu_snooping(domain);
540}
541
276dbf99 542static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 543{
544 struct dmar_drhd_unit *drhd = NULL;
545 int i;
546
547 for_each_drhd_unit(drhd) {
548 if (drhd->ignored)
549 continue;
276dbf99
DW		 550 if (segment != drhd->segment)
551 continue;
c7151a8d 552
924b6231 553 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 554 if (drhd->devices[i] &&
555 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 556 drhd->devices[i]->devfn == devfn)
557 return drhd->iommu;
4958c5dc
DW		 558 if (drhd->devices[i] &&
559 drhd->devices[i]->subordinate &&
924b6231
DW		 560 drhd->devices[i]->subordinate->number <= bus &&
561 drhd->devices[i]->subordinate->subordinate >= bus)
562 return drhd->iommu;
563 }
c7151a8d
WH		 564
565 if (drhd->include_all)
566 return drhd->iommu;
567 }
568
569 return NULL;
570}
571
5331fe6f
WH		 572static void domain_flush_cache(struct dmar_domain *domain,
573 void *addr, int size)
574{
575 if (!domain->iommu_coherency)
576 clflush_cache_range(addr, size);
577}
578
ba395927
KA		 579/* Gets context entry for a given bus and devfn */
580static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
581 u8 bus, u8 devfn)
582{
583 struct root_entry *root;
584 struct context_entry *context;
585 unsigned long phy_addr;
586 unsigned long flags;
587
588 spin_lock_irqsave(&iommu->lock, flags);
589 root = &iommu->root_entry[bus];
590 context = get_context_addr_from_root(root);
591 if (!context) {
592 context = (struct context_entry *)alloc_pgtable_page();
593 if (!context) {
594 spin_unlock_irqrestore(&iommu->lock, flags);
595 return NULL;
596 }
5b6985ce 597 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 598 phy_addr = virt_to_phys((void *)context);
599 set_root_value(root, phy_addr);
600 set_root_present(root);
601 __iommu_flush_cache(iommu, root, sizeof(*root));
602 }
603 spin_unlock_irqrestore(&iommu->lock, flags);
604 return &context[devfn];
605}
606
607static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
608{
609 struct root_entry *root;
610 struct context_entry *context;
611 int ret;
612 unsigned long flags;
613
614 spin_lock_irqsave(&iommu->lock, flags);
615 root = &iommu->root_entry[bus];
616 context = get_context_addr_from_root(root);
617 if (!context) {
618 ret = 0;
619 goto out;
620 }
c07e7d21 621 ret = context_present(&context[devfn]);
ba395927
KA		 622out:
623 spin_unlock_irqrestore(&iommu->lock, flags);
624 return ret;
625}
626
627static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
628{
629 struct root_entry *root;
630 struct context_entry *context;
631 unsigned long flags;
632
633 spin_lock_irqsave(&iommu->lock, flags);
634 root = &iommu->root_entry[bus];
635 context = get_context_addr_from_root(root);
636 if (context) {
c07e7d21 637 context_clear_entry(&context[devfn]);
ba395927
KA		 638 __iommu_flush_cache(iommu, &context[devfn], \
639 sizeof(*context));
640 }
641 spin_unlock_irqrestore(&iommu->lock, flags);
642}
643
644static void free_context_table(struct intel_iommu *iommu)
645{
646 struct root_entry *root;
647 int i;
648 unsigned long flags;
649 struct context_entry *context;
650
651 spin_lock_irqsave(&iommu->lock, flags);
652 if (!iommu->root_entry) {
653 goto out;
654 }
655 for (i = 0; i < ROOT_ENTRY_NR; i++) {
656 root = &iommu->root_entry[i];
657 context = get_context_addr_from_root(root);
658 if (context)
659 free_pgtable_page(context);
660 }
661 free_pgtable_page(iommu->root_entry);
662 iommu->root_entry = NULL;
663out:
664 spin_unlock_irqrestore(&iommu->lock, flags);
665}
666
667/* page table handling */
668#define LEVEL_STRIDE (9)
669#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
670
671static inline int agaw_to_level(int agaw)
672{
673 return agaw + 2;
674}
675
676static inline int agaw_to_width(int agaw)
677{
678 return 30 + agaw * LEVEL_STRIDE;
679
680}
681
682static inline int width_to_agaw(int width)
683{
684 return (width - 30) / LEVEL_STRIDE;
685}
686
687static inline unsigned int level_to_offset_bits(int level)
688{
6660c63a 689 return (level - 1) * LEVEL_STRIDE;
ba395927
KA		 690}
691
77dfa56c 692static inline int pfn_level_offset(unsigned long pfn, int level)
ba395927 693{
6660c63a 694 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
ba395927
KA		 695}
696
6660c63a 697static inline unsigned long level_mask(int level)
ba395927 698{
6660c63a 699 return -1UL << level_to_offset_bits(level);
ba395927
KA		 700}
701
6660c63a 702static inline unsigned long level_size(int level)
ba395927 703{
6660c63a 704 return 1UL << level_to_offset_bits(level);
ba395927
KA		 705}
706
6660c63a 707static inline unsigned long align_to_level(unsigned long pfn, int level)
ba395927 708{
6660c63a 709 return (pfn + level_size(level) - 1) & level_mask(level);
ba395927
KA		 710}
711
b026fd28
DW		 712static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
713 unsigned long pfn)
ba395927 714{
b026fd28 715 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 716 struct dma_pte *parent, *pte = NULL;
717 int level = agaw_to_level(domain->agaw);
718 int offset;
ba395927
KA		 719
720 BUG_ON(!domain->pgd);
b026fd28 721 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
ba395927
KA		 722 parent = domain->pgd;
723
ba395927
KA		 724 while (level > 0) {
725 void *tmp_page;
726
b026fd28 727 offset = pfn_level_offset(pfn, level);
ba395927
KA		 728 pte = &parent[offset];
729 if (level == 1)
730 break;
731
19c239ce 732 if (!dma_pte_present(pte)) {
c85994e4
DW		 733 uint64_t pteval;
734
ba395927
KA		 735 tmp_page = alloc_pgtable_page();
736
206a73c1 737 if (!tmp_page)
ba395927 738 return NULL;
206a73c1 739
c85994e4 740 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 741 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 742 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
743 /* Someone else set it while we were thinking; use theirs. */
744 free_pgtable_page(tmp_page);
745 } else {
746 dma_pte_addr(pte);
747 domain_flush_cache(domain, pte, sizeof(*pte));
748 }
ba395927 749 }
19c239ce 750 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 751 level--;
752 }
753
ba395927
KA		 754 return pte;
755}
756
757/* return address's pte at specific level */
90dcfb5e
DW		 758static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
759 unsigned long pfn,
760 int level)
ba395927
KA		 761{
762 struct dma_pte *parent, *pte = NULL;
763 int total = agaw_to_level(domain->agaw);
764 int offset;
765
766 parent = domain->pgd;
767 while (level <= total) {
90dcfb5e 768 offset = pfn_level_offset(pfn, total);
ba395927
KA		 769 pte = &parent[offset];
770 if (level == total)
771 return pte;
772
19c239ce 773 if (!dma_pte_present(pte))
ba395927 774 break;
19c239ce 775 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 776 total--;
777 }
778 return NULL;
779}
780
ba395927 781/* clear last level pte, a tlb flush should be followed */
595badf5
DW		 782static void dma_pte_clear_range(struct dmar_domain *domain,
783 unsigned long start_pfn,
784 unsigned long last_pfn)
ba395927 785{
04b18e65 786 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
310a5ab9 787 struct dma_pte *first_pte, *pte;
66eae846 788
04b18e65 789 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 790 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 791 BUG_ON(start_pfn > last_pfn);
ba395927 792
04b18e65 793 /* we don't need lock here; nobody else touches the iova range */
59c36286 794 do {
310a5ab9
DW		 795 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
796 if (!pte) {
797 start_pfn = align_to_level(start_pfn + 1, 2);
798 continue;
799 }
75e6bf96 800 do {
310a5ab9
DW		 801 dma_clear_pte(pte);
802 start_pfn++;
803 pte++;
75e6bf96
DW		 804 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
805
310a5ab9
DW		 806 domain_flush_cache(domain, first_pte,
807 (void *)pte - (void *)first_pte);
59c36286
DW		 808
809 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 810}
811
812/* free page table pages. last level pte should already be cleared */
813static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 814 unsigned long start_pfn,
815 unsigned long last_pfn)
ba395927 816{
6660c63a 817 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
f3a0a52f 818 struct dma_pte *first_pte, *pte;
ba395927
KA		 819 int total = agaw_to_level(domain->agaw);
820 int level;
6660c63a 821 unsigned long tmp;
ba395927 822
6660c63a
DW		 823 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
824 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 825 BUG_ON(start_pfn > last_pfn);
ba395927 826
f3a0a52f 827 /* We don't need lock here; nobody else touches the iova range */
ba395927
KA		 828 level = 2;
829 while (level <= total) {
6660c63a
DW		 830 tmp = align_to_level(start_pfn, level);
831
f3a0a52f 832 /* If we can't even clear one PTE at this level, we're done */
6660c63a 833 if (tmp + level_size(level) - 1 > last_pfn)
ba395927
KA		 834 return;
835
59c36286 836 do {
f3a0a52f
DW		 837 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
838 if (!pte) {
839 tmp = align_to_level(tmp + 1, level + 1);
840 continue;
841 }
75e6bf96 842 do {
6a43e574
DW		 843 if (dma_pte_present(pte)) {
844 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
845 dma_clear_pte(pte);
846 }
f3a0a52f
DW		 847 pte++;
848 tmp += level_size(level);
75e6bf96
DW		 849 } while (!first_pte_in_page(pte) &&
850 tmp + level_size(level) - 1 <= last_pfn);
851
f3a0a52f
DW		 852 domain_flush_cache(domain, first_pte,
853 (void *)pte - (void *)first_pte);
854
59c36286 855 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
ba395927
KA		 856 level++;
857 }
858 /* free pgd */
d794dc9b 859 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 860 free_pgtable_page(domain->pgd);
861 domain->pgd = NULL;
862 }
863}
864
865/* iommu handling */
866static int iommu_alloc_root_entry(struct intel_iommu *iommu)
867{
868 struct root_entry *root;
869 unsigned long flags;
870
871 root = (struct root_entry *)alloc_pgtable_page();
872 if (!root)
873 return -ENOMEM;
874
5b6985ce 875 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 876
877 spin_lock_irqsave(&iommu->lock, flags);
878 iommu->root_entry = root;
879 spin_unlock_irqrestore(&iommu->lock, flags);
880
881 return 0;
882}
883
ba395927
KA		 884static void iommu_set_root_entry(struct intel_iommu *iommu)
885{
886 void *addr;
c416daa9 887 u32 sts;
ba395927
KA		 888 unsigned long flag;
889
890 addr = iommu->root_entry;
891
892 spin_lock_irqsave(&iommu->register_lock, flag);
893 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
894
c416daa9 895 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 896
897 /* Make sure hardware complete it */
898 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 899 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927
KA		 900
901 spin_unlock_irqrestore(&iommu->register_lock, flag);
902}
903
904static void iommu_flush_write_buffer(struct intel_iommu *iommu)
905{
906 u32 val;
907 unsigned long flag;
908
9af88143 909 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 910 return;
ba395927
KA		 911
912 spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 913 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 914
915 /* Make sure hardware complete it */
916 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 917 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927
KA		 918
919 spin_unlock_irqrestore(&iommu->register_lock, flag);
920}
921
922/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 923static void __iommu_flush_context(struct intel_iommu *iommu,
924 u16 did, u16 source_id, u8 function_mask,
925 u64 type)
ba395927
KA		 926{
927 u64 val = 0;
928 unsigned long flag;
929
ba395927
KA		 930 switch (type) {
931 case DMA_CCMD_GLOBAL_INVL:
932 val = DMA_CCMD_GLOBAL_INVL;
933 break;
934 case DMA_CCMD_DOMAIN_INVL:
935 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
936 break;
937 case DMA_CCMD_DEVICE_INVL:
938 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
939 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
940 break;
941 default:
942 BUG();
943 }
944 val |= DMA_CCMD_ICC;
945
946 spin_lock_irqsave(&iommu->register_lock, flag);
947 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
948
949 /* Make sure hardware complete it */
950 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
951 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
952
953 spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 954}
955
ba395927 956/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 957static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
958 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 959{
960 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
961 u64 val = 0, val_iva = 0;
962 unsigned long flag;
963
ba395927
KA		 964 switch (type) {
965 case DMA_TLB_GLOBAL_FLUSH:
966 /* global flush doesn't need set IVA_REG */
967 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
968 break;
969 case DMA_TLB_DSI_FLUSH:
970 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
971 break;
972 case DMA_TLB_PSI_FLUSH:
973 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
974 /* Note: always flush non-leaf currently */
975 val_iva = size_order | addr;
976 break;
977 default:
978 BUG();
979 }
980 /* Note: set drain read/write */
981#if 0
982 /*
983 * This is probably to be super secure.. Looks like we can
984 * ignore it without any impact.
985 */
986 if (cap_read_drain(iommu->cap))
987 val |= DMA_TLB_READ_DRAIN;
988#endif
989 if (cap_write_drain(iommu->cap))
990 val |= DMA_TLB_WRITE_DRAIN;
991
992 spin_lock_irqsave(&iommu->register_lock, flag);
993 /* Note: Only uses first TLB reg currently */
994 if (val_iva)
995 dmar_writeq(iommu->reg + tlb_offset, val_iva);
996 dmar_writeq(iommu->reg + tlb_offset + 8, val);
997
998 /* Make sure hardware complete it */
999 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1000 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1001
1002 spin_unlock_irqrestore(&iommu->register_lock, flag);
1003
1004 /* check IOTLB invalidation granularity */
1005 if (DMA_TLB_IAIG(val) == 0)
1006 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1007 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1008 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1009 (unsigned long long)DMA_TLB_IIRG(type),
1010 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1011}
1012
93a23a72
YZ		 1013static struct device_domain_info *iommu_support_dev_iotlb(
1014 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1015{
1016 int found = 0;
1017 unsigned long flags;
1018 struct device_domain_info *info;
1019 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1020
1021 if (!ecap_dev_iotlb_support(iommu->ecap))
1022 return NULL;
1023
1024 if (!iommu->qi)
1025 return NULL;
1026
1027 spin_lock_irqsave(&device_domain_lock, flags);
1028 list_for_each_entry(info, &domain->devices, link)
1029 if (info->bus == bus && info->devfn == devfn) {
1030 found = 1;
1031 break;
1032 }
1033 spin_unlock_irqrestore(&device_domain_lock, flags);
1034
1035 if (!found || !info->dev)
1036 return NULL;
1037
1038 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1039 return NULL;
1040
1041 if (!dmar_find_matched_atsr_unit(info->dev))
1042 return NULL;
1043
1044 info->iommu = iommu;
1045
1046 return info;
1047}
1048
1049static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1050{
93a23a72
YZ		 1051 if (!info)
1052 return;
1053
1054 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1055}
1056
1057static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1058{
1059 if (!info->dev || !pci_ats_enabled(info->dev))
1060 return;
1061
1062 pci_disable_ats(info->dev);
1063}
1064
1065static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1066 u64 addr, unsigned mask)
1067{
1068 u16 sid, qdep;
1069 unsigned long flags;
1070 struct device_domain_info *info;
1071
1072 spin_lock_irqsave(&device_domain_lock, flags);
1073 list_for_each_entry(info, &domain->devices, link) {
1074 if (!info->dev || !pci_ats_enabled(info->dev))
1075 continue;
1076
1077 sid = info->bus << 8 | info->devfn;
1078 qdep = pci_ats_queue_depth(info->dev);
1079 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1080 }
1081 spin_unlock_irqrestore(&device_domain_lock, flags);
1082}
1083
1f0ef2aa 1084static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
03d6a246 1085 unsigned long pfn, unsigned int pages)
ba395927 1086{
9dd2fe89 1087 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1088 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1089
ba395927
KA		 1090 BUG_ON(pages == 0);
1091
ba395927 1092 /*
9dd2fe89
YZ		 1093 * Fallback to domain selective flush if no PSI support or the size is
1094 * too big.
ba395927
KA		 1095 * PSI requires page size to be 2 ^ x, and the base address is naturally
1096 * aligned to the size
1097 */
9dd2fe89
YZ		 1098 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1099 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1100 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1101 else
1102 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1103 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1104
1105 /*
1106 * In caching mode, domain ID 0 is reserved for non-present to present
1107 * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1108 */
1109 if (!cap_caching_mode(iommu->cap) || did)
93a23a72 1110 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1111}
1112
f8bab735 1113static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1114{
1115 u32 pmen;
1116 unsigned long flags;
1117
1118 spin_lock_irqsave(&iommu->register_lock, flags);
1119 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1120 pmen &= ~DMA_PMEN_EPM;
1121 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1122
1123 /* wait for the protected region status bit to clear */
1124 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1125 readl, !(pmen & DMA_PMEN_PRS), pmen);
1126
1127 spin_unlock_irqrestore(&iommu->register_lock, flags);
1128}
1129
ba395927
KA		 1130static int iommu_enable_translation(struct intel_iommu *iommu)
1131{
1132 u32 sts;
1133 unsigned long flags;
1134
1135 spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1136 iommu->gcmd |= DMA_GCMD_TE;
1137 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1138
1139 /* Make sure hardware complete it */
1140 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1141 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1142
ba395927
KA		 1143 spin_unlock_irqrestore(&iommu->register_lock, flags);
1144 return 0;
1145}
1146
1147static int iommu_disable_translation(struct intel_iommu *iommu)
1148{
1149 u32 sts;
1150 unsigned long flag;
1151
1152 spin_lock_irqsave(&iommu->register_lock, flag);
1153 iommu->gcmd &= ~DMA_GCMD_TE;
1154 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1155
1156 /* Make sure hardware complete it */
1157 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1158 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927
KA		 1159
1160 spin_unlock_irqrestore(&iommu->register_lock, flag);
1161 return 0;
1162}
1163
3460a6d9 1164
ba395927
KA		 1165static int iommu_init_domains(struct intel_iommu *iommu)
1166{
1167 unsigned long ndomains;
1168 unsigned long nlongs;
1169
1170 ndomains = cap_ndoms(iommu->cap);
1171 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1172 nlongs = BITS_TO_LONGS(ndomains);
1173
94a91b50
DD		 1174 spin_lock_init(&iommu->lock);
1175
ba395927
KA		 1176 /* TBD: there might be 64K domains,
1177 * consider other allocation for future chip
1178 */
1179 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1180 if (!iommu->domain_ids) {
1181 printk(KERN_ERR "Allocating domain id array failed\n");
1182 return -ENOMEM;
1183 }
1184 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1185 GFP_KERNEL);
1186 if (!iommu->domains) {
1187 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1188 return -ENOMEM;
1189 }
1190
1191 /*
1192 * if Caching mode is set, then invalid translations are tagged
1193 * with domainid 0. Hence we need to pre-allocate it.
1194 */
1195 if (cap_caching_mode(iommu->cap))
1196 set_bit(0, iommu->domain_ids);
1197 return 0;
1198}
ba395927 1199
ba395927
KA		 1200
1201static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1202static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1203
1204void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1205{
1206 struct dmar_domain *domain;
1207 int i;
c7151a8d 1208 unsigned long flags;
ba395927 1209
94a91b50
DD		 1210 if ((iommu->domains) && (iommu->domain_ids)) {
1211 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1212 for (; i < cap_ndoms(iommu->cap); ) {
1213 domain = iommu->domains[i];
1214 clear_bit(i, iommu->domain_ids);
1215
1216 spin_lock_irqsave(&domain->iommu_lock, flags);
1217 if (--domain->iommu_count == 0) {
1218 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1219 vm_domain_exit(domain);
1220 else
1221 domain_exit(domain);
1222 }
1223 spin_unlock_irqrestore(&domain->iommu_lock, flags);
c7151a8d 1224
94a91b50
DD		 1225 i = find_next_bit(iommu->domain_ids,
1226 cap_ndoms(iommu->cap), i+1);
5e98c4b1 1227 }
ba395927
KA		 1228 }
1229
1230 if (iommu->gcmd & DMA_GCMD_TE)
1231 iommu_disable_translation(iommu);
1232
1233 if (iommu->irq) {
1234 set_irq_data(iommu->irq, NULL);
1235 /* This will mask the irq */
1236 free_irq(iommu->irq, iommu);
1237 destroy_irq(iommu->irq);
1238 }
1239
1240 kfree(iommu->domains);
1241 kfree(iommu->domain_ids);
1242
d9630fe9
WH		 1243 g_iommus[iommu->seq_id] = NULL;
1244
1245 /* if all iommus are freed, free g_iommus */
1246 for (i = 0; i < g_num_of_iommus; i++) {
1247 if (g_iommus[i])
1248 break;
1249 }
1250
1251 if (i == g_num_of_iommus)
1252 kfree(g_iommus);
1253
ba395927
KA		 1254 /* free context mapping */
1255 free_context_table(iommu);
ba395927
KA		 1256}
1257
2c2e2c38 1258static struct dmar_domain *alloc_domain(void)
ba395927 1259{
ba395927 1260 struct dmar_domain *domain;
ba395927
KA		 1261
1262 domain = alloc_domain_mem();
1263 if (!domain)
1264 return NULL;
1265
2c2e2c38
FY		 1266 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1267 domain->flags = 0;
1268
1269 return domain;
1270}
1271
1272static int iommu_attach_domain(struct dmar_domain *domain,
1273 struct intel_iommu *iommu)
1274{
1275 int num;
1276 unsigned long ndomains;
1277 unsigned long flags;
1278
ba395927
KA		 1279 ndomains = cap_ndoms(iommu->cap);
1280
1281 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1282
ba395927
KA		 1283 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1284 if (num >= ndomains) {
1285 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1286 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1287 return -ENOMEM;
ba395927
KA		 1288 }
1289
ba395927 1290 domain->id = num;
2c2e2c38 1291 set_bit(num, iommu->domain_ids);
8c11e798 1292 set_bit(iommu->seq_id, &domain->iommu_bmp);
ba395927
KA		 1293 iommu->domains[num] = domain;
1294 spin_unlock_irqrestore(&iommu->lock, flags);
1295
2c2e2c38 1296 return 0;
ba395927
KA		 1297}
1298
2c2e2c38
FY		 1299static void iommu_detach_domain(struct dmar_domain *domain,
1300 struct intel_iommu *iommu)
ba395927
KA		 1301{
1302 unsigned long flags;
2c2e2c38
FY		 1303 int num, ndomains;
1304 int found = 0;
ba395927 1305
8c11e798 1306 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38
FY		 1307 ndomains = cap_ndoms(iommu->cap);
1308 num = find_first_bit(iommu->domain_ids, ndomains);
1309 for (; num < ndomains; ) {
1310 if (iommu->domains[num] == domain) {
1311 found = 1;
1312 break;
1313 }
1314 num = find_next_bit(iommu->domain_ids,
1315 cap_ndoms(iommu->cap), num+1);
1316 }
1317
1318 if (found) {
1319 clear_bit(num, iommu->domain_ids);
1320 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1321 iommu->domains[num] = NULL;
1322 }
8c11e798 1323 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1324}
1325
1326static struct iova_domain reserved_iova_list;
8a443df4 1327static struct lock_class_key reserved_rbtree_key;
ba395927
KA		 1328
1329static void dmar_init_reserved_ranges(void)
1330{
1331 struct pci_dev *pdev = NULL;
1332 struct iova *iova;
1333 int i;
ba395927 1334
f661197e 1335 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1336
8a443df4
MG		 1337 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1338 &reserved_rbtree_key);
1339
ba395927
KA		 1340 /* IOAPIC ranges shouldn't be accessed by DMA */
1341 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1342 IOVA_PFN(IOAPIC_RANGE_END));
1343 if (!iova)
1344 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1345
1346 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1347 for_each_pci_dev(pdev) {
1348 struct resource *r;
1349
1350 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1351 r = &pdev->resource[i];
1352 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1353 continue;
1a4a4551
DW		 1354 iova = reserve_iova(&reserved_iova_list,
1355 IOVA_PFN(r->start),
1356 IOVA_PFN(r->end));
ba395927
KA		 1357 if (!iova)
1358 printk(KERN_ERR "Reserve iova failed\n");
1359 }
1360 }
1361
1362}
1363
1364static void domain_reserve_special_ranges(struct dmar_domain *domain)
1365{
1366 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1367}
1368
1369static inline int guestwidth_to_adjustwidth(int gaw)
1370{
1371 int agaw;
1372 int r = (gaw - 12) % 9;
1373
1374 if (r == 0)
1375 agaw = gaw;
1376 else
1377 agaw = gaw + 9 - r;
1378 if (agaw > 64)
1379 agaw = 64;
1380 return agaw;
1381}
1382
1383static int domain_init(struct dmar_domain *domain, int guest_width)
1384{
1385 struct intel_iommu *iommu;
1386 int adjust_width, agaw;
1387 unsigned long sagaw;
1388
f661197e 1389 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1390 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1391
1392 domain_reserve_special_ranges(domain);
1393
1394 /* calculate AGAW */
8c11e798 1395 iommu = domain_get_iommu(domain);
ba395927
KA		 1396 if (guest_width > cap_mgaw(iommu->cap))
1397 guest_width = cap_mgaw(iommu->cap);
1398 domain->gaw = guest_width;
1399 adjust_width = guestwidth_to_adjustwidth(guest_width);
1400 agaw = width_to_agaw(adjust_width);
1401 sagaw = cap_sagaw(iommu->cap);
1402 if (!test_bit(agaw, &sagaw)) {
1403 /* hardware doesn't support it, choose a bigger one */
1404 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1405 agaw = find_next_bit(&sagaw, 5, agaw);
1406 if (agaw >= 5)
1407 return -ENODEV;
1408 }
1409 domain->agaw = agaw;
1410 INIT_LIST_HEAD(&domain->devices);
1411
8e604097
WH		 1412 if (ecap_coherent(iommu->ecap))
1413 domain->iommu_coherency = 1;
1414 else
1415 domain->iommu_coherency = 0;
1416
58c610bd
SY		 1417 if (ecap_sc_support(iommu->ecap))
1418 domain->iommu_snooping = 1;
1419 else
1420 domain->iommu_snooping = 0;
1421
c7151a8d
WH		 1422 domain->iommu_count = 1;
1423
ba395927
KA		 1424 /* always allocate the top pgd */
1425 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1426 if (!domain->pgd)
1427 return -ENOMEM;
5b6985ce 1428 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1429 return 0;
1430}
1431
1432static void domain_exit(struct dmar_domain *domain)
1433{
2c2e2c38
FY		 1434 struct dmar_drhd_unit *drhd;
1435 struct intel_iommu *iommu;
ba395927
KA		 1436
1437 /* Domain 0 is reserved, so dont process it */
1438 if (!domain)
1439 return;
1440
1441 domain_remove_dev_info(domain);
1442 /* destroy iovas */
1443 put_iova_domain(&domain->iovad);
ba395927
KA		 1444
1445 /* clear ptes */
595badf5 1446 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1447
1448 /* free page tables */
d794dc9b 1449 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1450
2c2e2c38
FY		 1451 for_each_active_iommu(iommu, drhd)
1452 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1453 iommu_detach_domain(domain, iommu);
1454
ba395927
KA		 1455 free_domain_mem(domain);
1456}
1457
4ed0d3e6
FY		 1458static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1459 u8 bus, u8 devfn, int translation)
ba395927
KA		 1460{
1461 struct context_entry *context;
ba395927 1462 unsigned long flags;
5331fe6f 1463 struct intel_iommu *iommu;
ea6606b0
WH		 1464 struct dma_pte *pgd;
1465 unsigned long num;
1466 unsigned long ndomains;
1467 int id;
1468 int agaw;
93a23a72 1469 struct device_domain_info *info = NULL;
ba395927
KA		 1470
1471 pr_debug("Set context mapping for %02x:%02x.%d\n",
1472 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1473
ba395927 1474 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1475 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1476 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1477
276dbf99 1478 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1479 if (!iommu)
1480 return -ENODEV;
1481
ba395927
KA		 1482 context = device_to_context_entry(iommu, bus, devfn);
1483 if (!context)
1484 return -ENOMEM;
1485 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1486 if (context_present(context)) {
ba395927
KA		 1487 spin_unlock_irqrestore(&iommu->lock, flags);
1488 return 0;
1489 }
1490
ea6606b0
WH		 1491 id = domain->id;
1492 pgd = domain->pgd;
1493
2c2e2c38
FY		 1494 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1495 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1496 int found = 0;
1497
1498 /* find an available domain id for this device in iommu */
1499 ndomains = cap_ndoms(iommu->cap);
1500 num = find_first_bit(iommu->domain_ids, ndomains);
1501 for (; num < ndomains; ) {
1502 if (iommu->domains[num] == domain) {
1503 id = num;
1504 found = 1;
1505 break;
1506 }
1507 num = find_next_bit(iommu->domain_ids,
1508 cap_ndoms(iommu->cap), num+1);
1509 }
1510
1511 if (found == 0) {
1512 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1513 if (num >= ndomains) {
1514 spin_unlock_irqrestore(&iommu->lock, flags);
1515 printk(KERN_ERR "IOMMU: no free domain ids\n");
1516 return -EFAULT;
1517 }
1518
1519 set_bit(num, iommu->domain_ids);
1520 iommu->domains[num] = domain;
1521 id = num;
1522 }
1523
1524 /* Skip top levels of page tables for
1525 * iommu which has less agaw than default.
1526 */
1527 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1528 pgd = phys_to_virt(dma_pte_addr(pgd));
1529 if (!dma_pte_present(pgd)) {
1530 spin_unlock_irqrestore(&iommu->lock, flags);
1531 return -ENOMEM;
1532 }
1533 }
1534 }
1535
1536 context_set_domain_id(context, id);
4ed0d3e6 1537
93a23a72
YZ		 1538 if (translation != CONTEXT_TT_PASS_THROUGH) {
1539 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1540 translation = info ? CONTEXT_TT_DEV_IOTLB :
1541 CONTEXT_TT_MULTI_LEVEL;
1542 }
4ed0d3e6
FY		 1543 /*
1544 * In pass through mode, AW must be programmed to indicate the largest
1545 * AGAW value supported by hardware. And ASR is ignored by hardware.
1546 */
93a23a72 1547 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1548 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1549 else {
1550 context_set_address_root(context, virt_to_phys(pgd));
1551 context_set_address_width(context, iommu->agaw);
1552 }
4ed0d3e6
FY		 1553
1554 context_set_translation_type(context, translation);
c07e7d21
MM		 1555 context_set_fault_enable(context);
1556 context_set_present(context);
5331fe6f 1557 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1558
4c25a2c1
DW		 1559 /*
1560 * It's a non-present to present mapping. If hardware doesn't cache
1561 * non-present entry we only need to flush the write-buffer. If the
1562 * _does_ cache non-present entries, then it does so in the special
1563 * domain #0, which we have to flush:
1564 */
1565 if (cap_caching_mode(iommu->cap)) {
1566 iommu->flush.flush_context(iommu, 0,
1567 (((u16)bus) << 8) | devfn,
1568 DMA_CCMD_MASK_NOBIT,
1569 DMA_CCMD_DEVICE_INVL);
1f0ef2aa 1570 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1571 } else {
ba395927 1572 iommu_flush_write_buffer(iommu);
4c25a2c1 1573 }
93a23a72 1574 iommu_enable_dev_iotlb(info);
ba395927 1575 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1576
1577 spin_lock_irqsave(&domain->iommu_lock, flags);
1578 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1579 domain->iommu_count++;
58c610bd 1580 domain_update_iommu_cap(domain);
c7151a8d
WH		 1581 }
1582 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1583 return 0;
1584}
1585
1586static int
4ed0d3e6
FY		 1587domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1588 int translation)
ba395927
KA		 1589{
1590 int ret;
1591 struct pci_dev *tmp, *parent;
1592
276dbf99 1593 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1594 pdev->bus->number, pdev->devfn,
1595 translation);
ba395927
KA		 1596 if (ret)
1597 return ret;
1598
1599 /* dependent device mapping */
1600 tmp = pci_find_upstream_pcie_bridge(pdev);
1601 if (!tmp)
1602 return 0;
1603 /* Secondary interface's bus number and devfn 0 */
1604 parent = pdev->bus->self;
1605 while (parent != tmp) {
276dbf99
DW		 1606 ret = domain_context_mapping_one(domain,
1607 pci_domain_nr(parent->bus),
1608 parent->bus->number,
4ed0d3e6 1609 parent->devfn, translation);
ba395927
KA		 1610 if (ret)
1611 return ret;
1612 parent = parent->bus->self;
1613 }
5f4d91a1 1614 if (pci_is_pcie(tmp)) /* this is a PCIE-to-PCI bridge */
ba395927 1615 return domain_context_mapping_one(domain,
276dbf99 1616 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1617 tmp->subordinate->number, 0,
1618 translation);
ba395927
KA		 1619 else /* this is a legacy PCI bridge */
1620 return domain_context_mapping_one(domain,
276dbf99
DW		 1621 pci_domain_nr(tmp->bus),
1622 tmp->bus->number,
4ed0d3e6
FY		 1623 tmp->devfn,
1624 translation);
ba395927
KA		 1625}
1626
5331fe6f 1627static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1628{
1629 int ret;
1630 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1631 struct intel_iommu *iommu;
1632
276dbf99
DW		 1633 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1634 pdev->devfn);
5331fe6f
WH		 1635 if (!iommu)
1636 return -ENODEV;
ba395927 1637
276dbf99 1638 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1639 if (!ret)
1640 return ret;
1641 /* dependent device mapping */
1642 tmp = pci_find_upstream_pcie_bridge(pdev);
1643 if (!tmp)
1644 return ret;
1645 /* Secondary interface's bus number and devfn 0 */
1646 parent = pdev->bus->self;
1647 while (parent != tmp) {
8c11e798 1648 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1649 parent->devfn);
ba395927
KA		 1650 if (!ret)
1651 return ret;
1652 parent = parent->bus->self;
1653 }
5f4d91a1 1654 if (pci_is_pcie(tmp))
276dbf99
DW		 1655 return device_context_mapped(iommu, tmp->subordinate->number,
1656 0);
ba395927 1657 else
276dbf99
DW		 1658 return device_context_mapped(iommu, tmp->bus->number,
1659 tmp->devfn);
ba395927
KA		 1660}
1661
f532959b
FY		 1662/* Returns a number of VTD pages, but aligned to MM page size */
1663static inline unsigned long aligned_nrpages(unsigned long host_addr,
1664 size_t size)
1665{
1666 host_addr &= ~PAGE_MASK;
1667 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1668}
1669
9051aa02
DW		 1670static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1671 struct scatterlist *sg, unsigned long phys_pfn,
1672 unsigned long nr_pages, int prot)
e1605495
DW		 1673{
1674 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1675 phys_addr_t uninitialized_var(pteval);
e1605495 1676 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1677 unsigned long sg_res;
e1605495
DW		 1678
1679 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1680
1681 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1682 return -EINVAL;
1683
1684 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1685
9051aa02
DW		 1686 if (sg)
1687 sg_res = 0;
1688 else {
1689 sg_res = nr_pages + 1;
1690 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1691 }
1692
e1605495 1693 while (nr_pages--) {
c85994e4
DW		 1694 uint64_t tmp;
1695
e1605495 1696 if (!sg_res) {
f532959b 1697 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1698 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1699 sg->dma_length = sg->length;
1700 pteval = page_to_phys(sg_page(sg)) | prot;
1701 }
1702 if (!pte) {
1703 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1704 if (!pte)
1705 return -ENOMEM;
1706 }
1707 /* We don't need lock here, nobody else
1708 * touches the iova range
1709 */
7766a3fb 1710 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1711 if (tmp) {
1bf20f0d 1712 static int dumps = 5;
c85994e4
DW		 1713 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1714 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1715 if (dumps) {
1716 dumps--;
1717 debug_dma_dump_mappings(NULL);
1718 }
1719 WARN_ON(1);
1720 }
e1605495 1721 pte++;
75e6bf96 1722 if (!nr_pages || first_pte_in_page(pte)) {
e1605495
DW		 1723 domain_flush_cache(domain, first_pte,
1724 (void *)pte - (void *)first_pte);
1725 pte = NULL;
1726 }
1727 iov_pfn++;
1728 pteval += VTD_PAGE_SIZE;
1729 sg_res--;
1730 if (!sg_res)
1731 sg = sg_next(sg);
1732 }
1733 return 0;
1734}
1735
9051aa02
DW		 1736static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1737 struct scatterlist *sg, unsigned long nr_pages,
1738 int prot)
ba395927 1739{
9051aa02
DW		 1740 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1741}
6f6a00e4 1742
9051aa02
DW		 1743static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1744 unsigned long phys_pfn, unsigned long nr_pages,
1745 int prot)
1746{
1747 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1748}
1749
c7151a8d 1750static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1751{
c7151a8d
WH		 1752 if (!iommu)
1753 return;
8c11e798
WH		 1754
1755 clear_context_table(iommu, bus, devfn);
1756 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1757 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1758 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1759}
1760
1761static void domain_remove_dev_info(struct dmar_domain *domain)
1762{
1763 struct device_domain_info *info;
1764 unsigned long flags;
c7151a8d 1765 struct intel_iommu *iommu;
ba395927
KA		 1766
1767 spin_lock_irqsave(&device_domain_lock, flags);
1768 while (!list_empty(&domain->devices)) {
1769 info = list_entry(domain->devices.next,
1770 struct device_domain_info, link);
1771 list_del(&info->link);
1772 list_del(&info->global);
1773 if (info->dev)
358dd8ac 1774 info->dev->dev.archdata.iommu = NULL;
ba395927
KA		 1775 spin_unlock_irqrestore(&device_domain_lock, flags);
1776
93a23a72 1777 iommu_disable_dev_iotlb(info);
276dbf99 1778 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1779 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1780 free_devinfo_mem(info);
1781
1782 spin_lock_irqsave(&device_domain_lock, flags);
1783 }
1784 spin_unlock_irqrestore(&device_domain_lock, flags);
1785}
1786
1787/*
1788 * find_domain
358dd8ac 1789 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1790 */
38717946 1791static struct dmar_domain *
ba395927
KA		 1792find_domain(struct pci_dev *pdev)
1793{
1794 struct device_domain_info *info;
1795
1796 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1797 info = pdev->dev.archdata.iommu;
ba395927
KA		 1798 if (info)
1799 return info->domain;
1800 return NULL;
1801}
1802
ba395927
KA		 1803/* domain is initialized */
1804static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1805{
1806 struct dmar_domain *domain, *found = NULL;
1807 struct intel_iommu *iommu;
1808 struct dmar_drhd_unit *drhd;
1809 struct device_domain_info *info, *tmp;
1810 struct pci_dev *dev_tmp;
1811 unsigned long flags;
1812 int bus = 0, devfn = 0;
276dbf99 1813 int segment;
2c2e2c38 1814 int ret;
ba395927
KA		 1815
1816 domain = find_domain(pdev);
1817 if (domain)
1818 return domain;
1819
276dbf99
DW		 1820 segment = pci_domain_nr(pdev->bus);
1821
ba395927
KA		 1822 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1823 if (dev_tmp) {
5f4d91a1 1824 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1825 bus = dev_tmp->subordinate->number;
1826 devfn = 0;
1827 } else {
1828 bus = dev_tmp->bus->number;
1829 devfn = dev_tmp->devfn;
1830 }
1831 spin_lock_irqsave(&device_domain_lock, flags);
1832 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1833 if (info->segment == segment &&
1834 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1835 found = info->domain;
1836 break;
1837 }
1838 }
1839 spin_unlock_irqrestore(&device_domain_lock, flags);
1840 /* pcie-pci bridge already has a domain, uses it */
1841 if (found) {
1842 domain = found;
1843 goto found_domain;
1844 }
1845 }
1846
2c2e2c38
FY		 1847 domain = alloc_domain();
1848 if (!domain)
1849 goto error;
1850
ba395927
KA		 1851 /* Allocate new domain for the device */
1852 drhd = dmar_find_matched_drhd_unit(pdev);
1853 if (!drhd) {
1854 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1855 pci_name(pdev));
1856 return NULL;
1857 }
1858 iommu = drhd->iommu;
1859
2c2e2c38
FY		 1860 ret = iommu_attach_domain(domain, iommu);
1861 if (ret) {
1862 domain_exit(domain);
ba395927 1863 goto error;
2c2e2c38 1864 }
ba395927
KA		 1865
1866 if (domain_init(domain, gaw)) {
1867 domain_exit(domain);
1868 goto error;
1869 }
1870
1871 /* register pcie-to-pci device */
1872 if (dev_tmp) {
1873 info = alloc_devinfo_mem();
1874 if (!info) {
1875 domain_exit(domain);
1876 goto error;
1877 }
276dbf99 1878 info->segment = segment;
ba395927
KA		 1879 info->bus = bus;
1880 info->devfn = devfn;
1881 info->dev = NULL;
1882 info->domain = domain;
1883 /* This domain is shared by devices under p2p bridge */
3b5410e7 1884 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 1885
1886 /* pcie-to-pci bridge already has a domain, uses it */
1887 found = NULL;
1888 spin_lock_irqsave(&device_domain_lock, flags);
1889 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 1890 if (tmp->segment == segment &&
1891 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 1892 found = tmp->domain;
1893 break;
1894 }
1895 }
1896 if (found) {
1897 free_devinfo_mem(info);
1898 domain_exit(domain);
1899 domain = found;
1900 } else {
1901 list_add(&info->link, &domain->devices);
1902 list_add(&info->global, &device_domain_list);
1903 }
1904 spin_unlock_irqrestore(&device_domain_lock, flags);
1905 }
1906
1907found_domain:
1908 info = alloc_devinfo_mem();
1909 if (!info)
1910 goto error;
276dbf99 1911 info->segment = segment;
ba395927
KA		 1912 info->bus = pdev->bus->number;
1913 info->devfn = pdev->devfn;
1914 info->dev = pdev;
1915 info->domain = domain;
1916 spin_lock_irqsave(&device_domain_lock, flags);
1917 /* somebody is fast */
1918 found = find_domain(pdev);
1919 if (found != NULL) {
1920 spin_unlock_irqrestore(&device_domain_lock, flags);
1921 if (found != domain) {
1922 domain_exit(domain);
1923 domain = found;
1924 }
1925 free_devinfo_mem(info);
1926 return domain;
1927 }
1928 list_add(&info->link, &domain->devices);
1929 list_add(&info->global, &device_domain_list);
358dd8ac 1930 pdev->dev.archdata.iommu = info;
ba395927
KA		 1931 spin_unlock_irqrestore(&device_domain_lock, flags);
1932 return domain;
1933error:
1934 /* recheck it here, maybe others set it */
1935 return find_domain(pdev);
1936}
1937
2c2e2c38 1938static int iommu_identity_mapping;
e0fc7e0b
DW		 1939#define IDENTMAP_ALL 1
1940#define IDENTMAP_GFX 2
1941#define IDENTMAP_AZALIA 4
2c2e2c38 1942
b213203e
DW		 1943static int iommu_domain_identity_map(struct dmar_domain *domain,
1944 unsigned long long start,
1945 unsigned long long end)
ba395927 1946{
c5395d5c
DW		 1947 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1948 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1949
1950 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1951 dma_to_mm_pfn(last_vpfn))) {
ba395927 1952 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 1953 return -ENOMEM;
ba395927
KA		 1954 }
1955
c5395d5c
DW		 1956 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1957 start, end, domain->id);
ba395927
KA		 1958 /*
1959 * RMRR range might have overlap with physical memory range,
1960 * clear it first
1961 */
c5395d5c 1962 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 1963
c5395d5c
DW		 1964 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1965 last_vpfn - first_vpfn + 1,
61df7443 1966 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 1967}
1968
1969static int iommu_prepare_identity_map(struct pci_dev *pdev,
1970 unsigned long long start,
1971 unsigned long long end)
1972{
1973 struct dmar_domain *domain;
1974 int ret;
1975
c7ab48d2 1976 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 1977 if (!domain)
1978 return -ENOMEM;
1979
19943b0e
DW		 1980 /* For _hardware_ passthrough, don't bother. But for software
1981 passthrough, we do it anyway -- it may indicate a memory
1982 range which is reserved in E820, so which didn't get set
1983 up to start with in si_domain */
1984 if (domain == si_domain && hw_pass_through) {
1985 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1986 pci_name(pdev), start, end);
1987 return 0;
1988 }
1989
1990 printk(KERN_INFO
1991 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1992 pci_name(pdev), start, end);
2ff729f5
DW		 1993
1994 if (end >> agaw_to_width(domain->agaw)) {
1995 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1996 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1997 agaw_to_width(domain->agaw),
1998 dmi_get_system_info(DMI_BIOS_VENDOR),
1999 dmi_get_system_info(DMI_BIOS_VERSION),
2000 dmi_get_system_info(DMI_PRODUCT_VERSION));
2001 ret = -EIO;
2002 goto error;
2003 }
19943b0e 2004
b213203e 2005 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2006 if (ret)
2007 goto error;
2008
2009 /* context entry init */
4ed0d3e6 2010 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2011 if (ret)
2012 goto error;
2013
2014 return 0;
2015
2016 error:
ba395927
KA		 2017 domain_exit(domain);
2018 return ret;
ba395927
KA		 2019}
2020
2021static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2022 struct pci_dev *pdev)
2023{
358dd8ac 2024 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2025 return 0;
2026 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2027 rmrr->end_address + 1);
2028}
2029
49a0429e
KA		 2030#ifdef CONFIG_DMAR_FLOPPY_WA
2031static inline void iommu_prepare_isa(void)
2032{
2033 struct pci_dev *pdev;
2034 int ret;
2035
2036 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2037 if (!pdev)
2038 return;
2039
c7ab48d2 2040 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
49a0429e
KA		 2041 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2042
2043 if (ret)
c7ab48d2
DW		 2044 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2045 "floppy might not work\n");
49a0429e
KA		 2046
2047}
2048#else
2049static inline void iommu_prepare_isa(void)
2050{
2051 return;
2052}
2053#endif /* !CONFIG_DMAR_FLPY_WA */
2054
2c2e2c38 2055static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2
DW		 2056
2057static int __init si_domain_work_fn(unsigned long start_pfn,
2058 unsigned long end_pfn, void *datax)
2059{
2060 int *ret = datax;
2061
2062 *ret = iommu_domain_identity_map(si_domain,
2063 (uint64_t)start_pfn << PAGE_SHIFT,
2064 (uint64_t)end_pfn << PAGE_SHIFT);
2065 return *ret;
2066
2067}
2068
071e1374 2069static int __init si_domain_init(int hw)
2c2e2c38
FY		 2070{
2071 struct dmar_drhd_unit *drhd;
2072 struct intel_iommu *iommu;
c7ab48d2 2073 int nid, ret = 0;
2c2e2c38
FY		 2074
2075 si_domain = alloc_domain();
2076 if (!si_domain)
2077 return -EFAULT;
2078
c7ab48d2 2079 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2080
2081 for_each_active_iommu(iommu, drhd) {
2082 ret = iommu_attach_domain(si_domain, iommu);
2083 if (ret) {
2084 domain_exit(si_domain);
2085 return -EFAULT;
2086 }
2087 }
2088
2089 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2090 domain_exit(si_domain);
2091 return -EFAULT;
2092 }
2093
2094 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2095
19943b0e
DW		 2096 if (hw)
2097 return 0;
2098
c7ab48d2
DW		 2099 for_each_online_node(nid) {
2100 work_with_active_regions(nid, si_domain_work_fn, &ret);
2101 if (ret)
2102 return ret;
2103 }
2104
2c2e2c38
FY		 2105 return 0;
2106}
2107
2108static void domain_remove_one_dev_info(struct dmar_domain *domain,
2109 struct pci_dev *pdev);
2110static int identity_mapping(struct pci_dev *pdev)
2111{
2112 struct device_domain_info *info;
2113
2114 if (likely(!iommu_identity_mapping))
2115 return 0;
2116
2117
2118 list_for_each_entry(info, &si_domain->devices, link)
2119 if (info->dev == pdev)
2120 return 1;
2121 return 0;
2122}
2123
2124static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2125 struct pci_dev *pdev,
2126 int translation)
2c2e2c38
FY		 2127{
2128 struct device_domain_info *info;
2129 unsigned long flags;
5fe60f4e 2130 int ret;
2c2e2c38
FY		 2131
2132 info = alloc_devinfo_mem();
2133 if (!info)
2134 return -ENOMEM;
2135
5fe60f4e
DW		 2136 ret = domain_context_mapping(domain, pdev, translation);
2137 if (ret) {
2138 free_devinfo_mem(info);
2139 return ret;
2140 }
2141
2c2e2c38
FY		 2142 info->segment = pci_domain_nr(pdev->bus);
2143 info->bus = pdev->bus->number;
2144 info->devfn = pdev->devfn;
2145 info->dev = pdev;
2146 info->domain = domain;
2147
2148 spin_lock_irqsave(&device_domain_lock, flags);
2149 list_add(&info->link, &domain->devices);
2150 list_add(&info->global, &device_domain_list);
2151 pdev->dev.archdata.iommu = info;
2152 spin_unlock_irqrestore(&device_domain_lock, flags);
2153
2154 return 0;
2155}
2156
6941af28
DW		 2157static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2158{
e0fc7e0b
DW		 2159 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2160 return 1;
2161
2162 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2163 return 1;
2164
2165 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2166 return 0;
6941af28 2167
3dfc813d
DW		 2168 /*
2169 * We want to start off with all devices in the 1:1 domain, and
2170 * take them out later if we find they can't access all of memory.
2171 *
2172 * However, we can't do this for PCI devices behind bridges,
2173 * because all PCI devices behind the same bridge will end up
2174 * with the same source-id on their transactions.
2175 *
2176 * Practically speaking, we can't change things around for these
2177 * devices at run-time, because we can't be sure there'll be no
2178 * DMA transactions in flight for any of their siblings.
2179 *
2180 * So PCI devices (unless they're on the root bus) as well as
2181 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2182 * the 1:1 domain, just in _case_ one of their siblings turns out
2183 * not to be able to map all of memory.
2184 */
5f4d91a1 2185 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2186 if (!pci_is_root_bus(pdev->bus))
2187 return 0;
2188 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2189 return 0;
2190 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2191 return 0;
2192
2193 /*
2194 * At boot time, we don't yet know if devices will be 64-bit capable.
2195 * Assume that they will -- if they turn out not to be, then we can
2196 * take them out of the 1:1 domain later.
2197 */
6941af28
DW		 2198 if (!startup)
2199 return pdev->dma_mask > DMA_BIT_MASK(32);
2200
2201 return 1;
2202}
2203
071e1374 2204static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2205{
2c2e2c38
FY		 2206 struct pci_dev *pdev = NULL;
2207 int ret;
2208
19943b0e 2209 ret = si_domain_init(hw);
2c2e2c38
FY		 2210 if (ret)
2211 return -EFAULT;
2212
2c2e2c38 2213 for_each_pci_dev(pdev) {
6941af28 2214 if (iommu_should_identity_map(pdev, 1)) {
19943b0e
DW		 2215 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2216 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2217
5fe60f4e 2218 ret = domain_add_dev_info(si_domain, pdev,
19943b0e 2219 hw ? CONTEXT_TT_PASS_THROUGH :
62edf5dc
DW		 2220 CONTEXT_TT_MULTI_LEVEL);
2221 if (ret)
2222 return ret;
62edf5dc 2223 }
2c2e2c38
FY		 2224 }
2225
2226 return 0;
2227}
2228
2229int __init init_dmars(void)
ba395927
KA		 2230{
2231 struct dmar_drhd_unit *drhd;
2232 struct dmar_rmrr_unit *rmrr;
2233 struct pci_dev *pdev;
2234 struct intel_iommu *iommu;
9d783ba0 2235 int i, ret;
2c2e2c38 2236
ba395927
KA		 2237 /*
2238 * for each drhd
2239 * allocate root
2240 * initialize and program root entry to not present
2241 * endfor
2242 */
2243 for_each_drhd_unit(drhd) {
5e0d2a6f 2244 g_num_of_iommus++;
2245 /*
2246 * lock not needed as this is only incremented in the single
2247 * threaded kernel __init code path all other access are read
2248 * only
2249 */
2250 }
2251
d9630fe9
WH		 2252 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2253 GFP_KERNEL);
2254 if (!g_iommus) {
2255 printk(KERN_ERR "Allocating global iommu array failed\n");
2256 ret = -ENOMEM;
2257 goto error;
2258 }
2259
80b20dd8 2260 deferred_flush = kzalloc(g_num_of_iommus *
2261 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2262 if (!deferred_flush) {
5e0d2a6f 2263 ret = -ENOMEM;
2264 goto error;
2265 }
2266
5e0d2a6f 2267 for_each_drhd_unit(drhd) {
2268 if (drhd->ignored)
2269 continue;
1886e8a9
SS		 2270
2271 iommu = drhd->iommu;
d9630fe9 2272 g_iommus[iommu->seq_id] = iommu;
ba395927 2273
e61d98d8
SS		 2274 ret = iommu_init_domains(iommu);
2275 if (ret)
2276 goto error;
2277
ba395927
KA		 2278 /*
2279 * TBD:
2280 * we could share the same root & context tables
2281 * amoung all IOMMU's. Need to Split it later.
2282 */
2283 ret = iommu_alloc_root_entry(iommu);
2284 if (ret) {
2285 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2286 goto error;
2287 }
4ed0d3e6 2288 if (!ecap_pass_through(iommu->ecap))
19943b0e 2289 hw_pass_through = 0;
ba395927
KA		 2290 }
2291
1531a6a6
SS		 2292 /*
2293 * Start from the sane iommu hardware state.
2294 */
a77b67d4
YS		 2295 for_each_drhd_unit(drhd) {
2296 if (drhd->ignored)
2297 continue;
2298
2299 iommu = drhd->iommu;
1531a6a6
SS		 2300
2301 /*
2302 * If the queued invalidation is already initialized by us
2303 * (for example, while enabling interrupt-remapping) then
2304 * we got the things already rolling from a sane state.
2305 */
2306 if (iommu->qi)
2307 continue;
2308
2309 /*
2310 * Clear any previous faults.
2311 */
2312 dmar_fault(-1, iommu);
2313 /*
2314 * Disable queued invalidation if supported and already enabled
2315 * before OS handover.
2316 */
2317 dmar_disable_qi(iommu);
2318 }
2319
2320 for_each_drhd_unit(drhd) {
2321 if (drhd->ignored)
2322 continue;
2323
2324 iommu = drhd->iommu;
2325
a77b67d4
YS		 2326 if (dmar_enable_qi(iommu)) {
2327 /*
2328 * Queued Invalidate not enabled, use Register Based
2329 * Invalidate
2330 */
2331 iommu->flush.flush_context = __iommu_flush_context;
2332 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2333 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
b4e0f9eb
FT		 2334 "invalidation\n",
2335 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2336 } else {
2337 iommu->flush.flush_context = qi_flush_context;
2338 iommu->flush.flush_iotlb = qi_flush_iotlb;
2339 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
b4e0f9eb
FT		 2340 "invalidation\n",
2341 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2342 }
2343 }
2344
19943b0e 2345 if (iommu_pass_through)
e0fc7e0b
DW		 2346 iommu_identity_mapping |= IDENTMAP_ALL;
2347
19943b0e 2348#ifdef CONFIG_DMAR_BROKEN_GFX_WA
e0fc7e0b 2349 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2350#endif
e0fc7e0b
DW		 2351
2352 check_tylersburg_isoch();
2353
ba395927 2354 /*
19943b0e
DW		 2355 * If pass through is not set or not enabled, setup context entries for
2356 * identity mappings for rmrr, gfx, and isa and may fall back to static
2357 * identity mapping if iommu_identity_mapping is set.
ba395927 2358 */
19943b0e
DW		 2359 if (iommu_identity_mapping) {
2360 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2361 if (ret) {
19943b0e
DW		 2362 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2363 goto error;
ba395927
KA		 2364 }
2365 }
ba395927 2366 /*
19943b0e
DW		 2367 * For each rmrr
2368 * for each dev attached to rmrr
2369 * do
2370 * locate drhd for dev, alloc domain for dev
2371 * allocate free domain
2372 * allocate page table entries for rmrr
2373 * if context not allocated for bus
2374 * allocate and init context
2375 * set present in root table for this bus
2376 * init context with domain, translation etc
2377 * endfor
2378 * endfor
ba395927 2379 */
19943b0e
DW		 2380 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2381 for_each_rmrr_units(rmrr) {
2382 for (i = 0; i < rmrr->devices_cnt; i++) {
2383 pdev = rmrr->devices[i];
2384 /*
2385 * some BIOS lists non-exist devices in DMAR
2386 * table.
2387 */
2388 if (!pdev)
2389 continue;
2390 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2391 if (ret)
2392 printk(KERN_ERR
2393 "IOMMU: mapping reserved region failed\n");
ba395927 2394 }
4ed0d3e6 2395 }
49a0429e 2396
19943b0e
DW		 2397 iommu_prepare_isa();
2398
ba395927
KA		 2399 /*
2400 * for each drhd
2401 * enable fault log
2402 * global invalidate context cache
2403 * global invalidate iotlb
2404 * enable translation
2405 */
2406 for_each_drhd_unit(drhd) {
2407 if (drhd->ignored)
2408 continue;
2409 iommu = drhd->iommu;
ba395927
KA		 2410
2411 iommu_flush_write_buffer(iommu);
2412
3460a6d9
KA		 2413 ret = dmar_set_interrupt(iommu);
2414 if (ret)
2415 goto error;
2416
ba395927
KA		 2417 iommu_set_root_entry(iommu);
2418
4c25a2c1 2419 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2420 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2421
ba395927
KA		 2422 ret = iommu_enable_translation(iommu);
2423 if (ret)
2424 goto error;
b94996c9
DW		 2425
2426 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2427 }
2428
2429 return 0;
2430error:
2431 for_each_drhd_unit(drhd) {
2432 if (drhd->ignored)
2433 continue;
2434 iommu = drhd->iommu;
2435 free_iommu(iommu);
2436 }
d9630fe9 2437 kfree(g_iommus);
ba395927
KA		 2438 return ret;
2439}
2440
5a5e02a6 2441/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2442static struct iova *intel_alloc_iova(struct device *dev,
2443 struct dmar_domain *domain,
2444 unsigned long nrpages, uint64_t dma_mask)
ba395927 2445{
ba395927 2446 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2447 struct iova *iova = NULL;
ba395927 2448
875764de
DW		 2449 /* Restrict dma_mask to the width that the iommu can handle */
2450 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2451
2452 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2453 /*
2454 * First try to allocate an io virtual address in
284901a9 2455 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2456 * from higher range
ba395927 2457 */
875764de
DW		 2458 iova = alloc_iova(&domain->iovad, nrpages,
2459 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2460 if (iova)
2461 return iova;
2462 }
2463 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2464 if (unlikely(!iova)) {
2465 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2466 nrpages, pci_name(pdev));
f76aec76
KA		 2467 return NULL;
2468 }
2469
2470 return iova;
2471}
2472
147202aa 2473static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2474{
2475 struct dmar_domain *domain;
2476 int ret;
2477
2478 domain = get_domain_for_dev(pdev,
2479 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2480 if (!domain) {
2481 printk(KERN_ERR
2482 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2483 return NULL;
ba395927
KA		 2484 }
2485
2486 /* make sure context mapping is ok */
5331fe6f 2487 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2488 ret = domain_context_mapping(domain, pdev,
2489 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2490 if (ret) {
2491 printk(KERN_ERR
2492 "Domain context map for %s failed",
2493 pci_name(pdev));
4fe05bbc 2494 return NULL;
f76aec76 2495 }
ba395927
KA		 2496 }
2497
f76aec76
KA		 2498 return domain;
2499}
2500
147202aa
DW		 2501static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2502{
2503 struct device_domain_info *info;
2504
2505 /* No lock here, assumes no domain exit in normal case */
2506 info = dev->dev.archdata.iommu;
2507 if (likely(info))
2508 return info->domain;
2509
2510 return __get_valid_domain_for_dev(dev);
2511}
2512
2c2e2c38
FY		 2513static int iommu_dummy(struct pci_dev *pdev)
2514{
2515 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2516}
2517
2518/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2519static int iommu_no_mapping(struct device *dev)
2c2e2c38 2520{
73676832 2521 struct pci_dev *pdev;
2c2e2c38
FY		 2522 int found;
2523
73676832
DW		 2524 if (unlikely(dev->bus != &pci_bus_type))
2525 return 1;
2526
2527 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2528 if (iommu_dummy(pdev))
2529 return 1;
2530
2c2e2c38 2531 if (!iommu_identity_mapping)
1e4c64c4 2532 return 0;
2c2e2c38
FY		 2533
2534 found = identity_mapping(pdev);
2535 if (found) {
6941af28 2536 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2537 return 1;
2538 else {
2539 /*
2540 * 32 bit DMA is removed from si_domain and fall back
2541 * to non-identity mapping.
2542 */
2543 domain_remove_one_dev_info(si_domain, pdev);
2544 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2545 pci_name(pdev));
2546 return 0;
2547 }
2548 } else {
2549 /*
2550 * In case of a detached 64 bit DMA device from vm, the device
2551 * is put into si_domain for identity mapping.
2552 */
6941af28 2553 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2554 int ret;
5fe60f4e
DW		 2555 ret = domain_add_dev_info(si_domain, pdev,
2556 hw_pass_through ?
2557 CONTEXT_TT_PASS_THROUGH :
2558 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2559 if (!ret) {
2560 printk(KERN_INFO "64bit %s uses identity mapping\n",
2561 pci_name(pdev));
2562 return 1;
2563 }
2564 }
2565 }
2566
1e4c64c4 2567 return 0;
2c2e2c38
FY		 2568}
2569
bb9e6d65
FT		 2570static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2571 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2572{
2573 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2574 struct dmar_domain *domain;
5b6985ce 2575 phys_addr_t start_paddr;
f76aec76
KA		 2576 struct iova *iova;
2577 int prot = 0;
6865f0d1 2578 int ret;
8c11e798 2579 struct intel_iommu *iommu;
33041ec0 2580 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2581
2582 BUG_ON(dir == DMA_NONE);
2c2e2c38 2583
73676832 2584 if (iommu_no_mapping(hwdev))
6865f0d1 2585 return paddr;
f76aec76
KA		 2586
2587 domain = get_valid_domain_for_dev(pdev);
2588 if (!domain)
2589 return 0;
2590
8c11e798 2591 iommu = domain_get_iommu(domain);
88cb6a74 2592 size = aligned_nrpages(paddr, size);
f76aec76 2593
5a5e02a6
DW		 2594 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2595 pdev->dma_mask);
f76aec76
KA		 2596 if (!iova)
2597 goto error;
2598
ba395927
KA		 2599 /*
2600 * Check if DMAR supports zero-length reads on write only
2601 * mappings..
2602 */
2603 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2604 !cap_zlr(iommu->cap))
ba395927
KA		 2605 prot |= DMA_PTE_READ;
2606 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2607 prot |= DMA_PTE_WRITE;
2608 /*
6865f0d1 2609 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2610 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2611 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2612 * is not a big problem
2613 */
0ab36de2 2614 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2615 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2616 if (ret)
2617 goto error;
2618
1f0ef2aa
DW		 2619 /* it's a non-present to present mapping. Only flush if caching mode */
2620 if (cap_caching_mode(iommu->cap))
03d6a246 2621 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
1f0ef2aa 2622 else
8c11e798 2623 iommu_flush_write_buffer(iommu);
f76aec76 2624
03d6a246
DW		 2625 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2626 start_paddr += paddr & ~PAGE_MASK;
2627 return start_paddr;
ba395927 2628
ba395927 2629error:
f76aec76
KA		 2630 if (iova)
2631 __free_iova(&domain->iovad, iova);
4cf2e75d 2632 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2633 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2634 return 0;
2635}
2636
ffbbef5c
FT		 2637static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2638 unsigned long offset, size_t size,
2639 enum dma_data_direction dir,
2640 struct dma_attrs *attrs)
bb9e6d65 2641{
ffbbef5c
FT		 2642 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2643 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2644}
2645
5e0d2a6f 2646static void flush_unmaps(void)
2647{
80b20dd8 2648 int i, j;
5e0d2a6f 2649
5e0d2a6f 2650 timer_on = 0;
2651
2652 /* just flush them all */
2653 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2654 struct intel_iommu *iommu = g_iommus[i];
2655 if (!iommu)
2656 continue;
c42d9f32 2657
9dd2fe89
YZ		 2658 if (!deferred_flush[i].next)
2659 continue;
2660
2661 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2662 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2663 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2664 unsigned long mask;
2665 struct iova *iova = deferred_flush[i].iova[j];
2666
64de5af0 2667 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
93a23a72 2668 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
64de5af0 2669 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
93a23a72 2670 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2671 }
9dd2fe89 2672 deferred_flush[i].next = 0;
5e0d2a6f 2673 }
2674
5e0d2a6f 2675 list_size = 0;
5e0d2a6f 2676}
2677
2678static void flush_unmaps_timeout(unsigned long data)
2679{
80b20dd8 2680 unsigned long flags;
2681
2682 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2683 flush_unmaps();
80b20dd8 2684 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2685}
2686
2687static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2688{
2689 unsigned long flags;
80b20dd8 2690 int next, iommu_id;
8c11e798 2691 struct intel_iommu *iommu;
5e0d2a6f 2692
2693 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2694 if (list_size == HIGH_WATER_MARK)
2695 flush_unmaps();
2696
8c11e798
WH		 2697 iommu = domain_get_iommu(dom);
2698 iommu_id = iommu->seq_id;
c42d9f32 2699
80b20dd8 2700 next = deferred_flush[iommu_id].next;
2701 deferred_flush[iommu_id].domain[next] = dom;
2702 deferred_flush[iommu_id].iova[next] = iova;
2703 deferred_flush[iommu_id].next++;
5e0d2a6f 2704
2705 if (!timer_on) {
2706 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2707 timer_on = 1;
2708 }
2709 list_size++;
2710 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2711}
2712
ffbbef5c
FT		 2713static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2714 size_t size, enum dma_data_direction dir,
2715 struct dma_attrs *attrs)
ba395927 2716{
ba395927 2717 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2718 struct dmar_domain *domain;
d794dc9b 2719 unsigned long start_pfn, last_pfn;
ba395927 2720 struct iova *iova;
8c11e798 2721 struct intel_iommu *iommu;
ba395927 2722
73676832 2723 if (iommu_no_mapping(dev))
f76aec76 2724 return;
2c2e2c38 2725
ba395927
KA		 2726 domain = find_domain(pdev);
2727 BUG_ON(!domain);
2728
8c11e798
WH		 2729 iommu = domain_get_iommu(domain);
2730
ba395927 2731 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2732 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2733 (unsigned long long)dev_addr))
ba395927 2734 return;
ba395927 2735
d794dc9b
DW		 2736 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2737 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2738
d794dc9b
DW		 2739 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2740 pci_name(pdev), start_pfn, last_pfn);
ba395927 2741
f76aec76 2742 /* clear the whole page */
d794dc9b
DW		 2743 dma_pte_clear_range(domain, start_pfn, last_pfn);
2744
f76aec76 2745 /* free page tables */
d794dc9b
DW		 2746 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2747
5e0d2a6f 2748 if (intel_iommu_strict) {
03d6a246 2749 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
d794dc9b 2750 last_pfn - start_pfn + 1);
5e0d2a6f 2751 /* free iova */
2752 __free_iova(&domain->iovad, iova);
2753 } else {
2754 add_unmap(domain, iova);
2755 /*
2756 * queue up the release of the unmap to save the 1/6th of the
2757 * cpu used up by the iotlb flush operation...
2758 */
5e0d2a6f 2759 }
ba395927
KA		 2760}
2761
d7ab5c46
FT		 2762static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2763 dma_addr_t *dma_handle, gfp_t flags)
ba395927
KA		 2764{
2765 void *vaddr;
2766 int order;
2767
5b6985ce 2768 size = PAGE_ALIGN(size);
ba395927 2769 order = get_order(size);
e8bb910d
AW		 2770
2771 if (!iommu_no_mapping(hwdev))
2772 flags &= ~(GFP_DMA | GFP_DMA32);
2773 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2774 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2775 flags |= GFP_DMA;
2776 else
2777 flags |= GFP_DMA32;
2778 }
ba395927
KA		 2779
2780 vaddr = (void *)__get_free_pages(flags, order);
2781 if (!vaddr)
2782 return NULL;
2783 memset(vaddr, 0, size);
2784
bb9e6d65
FT		 2785 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2786 DMA_BIDIRECTIONAL,
2787 hwdev->coherent_dma_mask);
ba395927
KA		 2788 if (*dma_handle)
2789 return vaddr;
2790 free_pages((unsigned long)vaddr, order);
2791 return NULL;
2792}
2793
d7ab5c46
FT		 2794static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2795 dma_addr_t dma_handle)
ba395927
KA		 2796{
2797 int order;
2798
5b6985ce 2799 size = PAGE_ALIGN(size);
ba395927
KA		 2800 order = get_order(size);
2801
0db9b7ae 2802 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 2803 free_pages((unsigned long)vaddr, order);
2804}
2805
d7ab5c46
FT		 2806static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2807 int nelems, enum dma_data_direction dir,
2808 struct dma_attrs *attrs)
ba395927 2809{
ba395927
KA		 2810 struct pci_dev *pdev = to_pci_dev(hwdev);
2811 struct dmar_domain *domain;
d794dc9b 2812 unsigned long start_pfn, last_pfn;
f76aec76 2813 struct iova *iova;
8c11e798 2814 struct intel_iommu *iommu;
ba395927 2815
73676832 2816 if (iommu_no_mapping(hwdev))
ba395927
KA		 2817 return;
2818
2819 domain = find_domain(pdev);
8c11e798
WH		 2820 BUG_ON(!domain);
2821
2822 iommu = domain_get_iommu(domain);
ba395927 2823
c03ab37c 2824 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 2825 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2826 (unsigned long long)sglist[0].dma_address))
f76aec76 2827 return;
f76aec76 2828
d794dc9b
DW		 2829 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2830 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 2831
2832 /* clear the whole page */
d794dc9b
DW		 2833 dma_pte_clear_range(domain, start_pfn, last_pfn);
2834
f76aec76 2835 /* free page tables */
d794dc9b 2836 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 2837
acea0018
DW		 2838 if (intel_iommu_strict) {
2839 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2840 last_pfn - start_pfn + 1);
2841 /* free iova */
2842 __free_iova(&domain->iovad, iova);
2843 } else {
2844 add_unmap(domain, iova);
2845 /*
2846 * queue up the release of the unmap to save the 1/6th of the
2847 * cpu used up by the iotlb flush operation...
2848 */
2849 }
ba395927
KA		 2850}
2851
ba395927 2852static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 2853 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 2854{
2855 int i;
c03ab37c 2856 struct scatterlist *sg;
ba395927 2857
c03ab37c 2858 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 2859 BUG_ON(!sg_page(sg));
4cf2e75d 2860 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 2861 sg->dma_length = sg->length;
ba395927
KA		 2862 }
2863 return nelems;
2864}
2865
d7ab5c46
FT		 2866static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2867 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 2868{
ba395927 2869 int i;
ba395927
KA		 2870 struct pci_dev *pdev = to_pci_dev(hwdev);
2871 struct dmar_domain *domain;
f76aec76
KA		 2872 size_t size = 0;
2873 int prot = 0;
b536d24d 2874 size_t offset_pfn = 0;
f76aec76
KA		 2875 struct iova *iova = NULL;
2876 int ret;
c03ab37c 2877 struct scatterlist *sg;
b536d24d 2878 unsigned long start_vpfn;
8c11e798 2879 struct intel_iommu *iommu;
ba395927
KA		 2880
2881 BUG_ON(dir == DMA_NONE);
73676832 2882 if (iommu_no_mapping(hwdev))
c03ab37c 2883 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 2884
f76aec76
KA		 2885 domain = get_valid_domain_for_dev(pdev);
2886 if (!domain)
2887 return 0;
2888
8c11e798
WH		 2889 iommu = domain_get_iommu(domain);
2890
b536d24d 2891 for_each_sg(sglist, sg, nelems, i)
88cb6a74 2892 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 2893
5a5e02a6
DW		 2894 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2895 pdev->dma_mask);
f76aec76 2896 if (!iova) {
c03ab37c 2897 sglist->dma_length = 0;
f76aec76
KA		 2898 return 0;
2899 }
2900
2901 /*
2902 * Check if DMAR supports zero-length reads on write only
2903 * mappings..
2904 */
2905 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2906 !cap_zlr(iommu->cap))
f76aec76
KA		 2907 prot |= DMA_PTE_READ;
2908 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2909 prot |= DMA_PTE_WRITE;
2910
b536d24d 2911 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 2912
f532959b 2913 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 2914 if (unlikely(ret)) {
2915 /* clear the page */
2916 dma_pte_clear_range(domain, start_vpfn,
2917 start_vpfn + size - 1);
2918 /* free page tables */
2919 dma_pte_free_pagetable(domain, start_vpfn,
2920 start_vpfn + size - 1);
2921 /* free iova */
2922 __free_iova(&domain->iovad, iova);
2923 return 0;
ba395927
KA		 2924 }
2925
1f0ef2aa
DW		 2926 /* it's a non-present to present mapping. Only flush if caching mode */
2927 if (cap_caching_mode(iommu->cap))
03d6a246 2928 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
1f0ef2aa 2929 else
8c11e798 2930 iommu_flush_write_buffer(iommu);
1f0ef2aa 2931
ba395927
KA		 2932 return nelems;
2933}
2934
dfb805e8
FT		 2935static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2936{
2937 return !dma_addr;
2938}
2939
160c1d8e 2940struct dma_map_ops intel_dma_ops = {
ba395927
KA		 2941 .alloc_coherent = intel_alloc_coherent,
2942 .free_coherent = intel_free_coherent,
ba395927
KA		 2943 .map_sg = intel_map_sg,
2944 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 2945 .map_page = intel_map_page,
2946 .unmap_page = intel_unmap_page,
dfb805e8 2947 .mapping_error = intel_mapping_error,
ba395927
KA		 2948};
2949
2950static inline int iommu_domain_cache_init(void)
2951{
2952 int ret = 0;
2953
2954 iommu_domain_cache = kmem_cache_create("iommu_domain",
2955 sizeof(struct dmar_domain),
2956 0,
2957 SLAB_HWCACHE_ALIGN,
2958
2959 NULL);
2960 if (!iommu_domain_cache) {
2961 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2962 ret = -ENOMEM;
2963 }
2964
2965 return ret;
2966}
2967
2968static inline int iommu_devinfo_cache_init(void)
2969{
2970 int ret = 0;
2971
2972 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2973 sizeof(struct device_domain_info),
2974 0,
2975 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2976 NULL);
2977 if (!iommu_devinfo_cache) {
2978 printk(KERN_ERR "Couldn't create devinfo cache\n");
2979 ret = -ENOMEM;
2980 }
2981
2982 return ret;
2983}
2984
2985static inline int iommu_iova_cache_init(void)
2986{
2987 int ret = 0;
2988
2989 iommu_iova_cache = kmem_cache_create("iommu_iova",
2990 sizeof(struct iova),
2991 0,
2992 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2993 NULL);
2994 if (!iommu_iova_cache) {
2995 printk(KERN_ERR "Couldn't create iova cache\n");
2996 ret = -ENOMEM;
2997 }
2998
2999 return ret;
3000}
3001
3002static int __init iommu_init_mempool(void)
3003{
3004 int ret;
3005 ret = iommu_iova_cache_init();
3006 if (ret)
3007 return ret;
3008
3009 ret = iommu_domain_cache_init();
3010 if (ret)
3011 goto domain_error;
3012
3013 ret = iommu_devinfo_cache_init();
3014 if (!ret)
3015 return ret;
3016
3017 kmem_cache_destroy(iommu_domain_cache);
3018domain_error:
3019 kmem_cache_destroy(iommu_iova_cache);
3020
3021 return -ENOMEM;
3022}
3023
3024static void __init iommu_exit_mempool(void)
3025{
3026 kmem_cache_destroy(iommu_devinfo_cache);
3027 kmem_cache_destroy(iommu_domain_cache);
3028 kmem_cache_destroy(iommu_iova_cache);
3029
3030}
3031
ba395927
KA		 3032static void __init init_no_remapping_devices(void)
3033{
3034 struct dmar_drhd_unit *drhd;
3035
3036 for_each_drhd_unit(drhd) {
3037 if (!drhd->include_all) {
3038 int i;
3039 for (i = 0; i < drhd->devices_cnt; i++)
3040 if (drhd->devices[i] != NULL)
3041 break;
3042 /* ignore DMAR unit if no pci devices exist */
3043 if (i == drhd->devices_cnt)
3044 drhd->ignored = 1;
3045 }
3046 }
3047
3048 if (dmar_map_gfx)
3049 return;
3050
3051 for_each_drhd_unit(drhd) {
3052 int i;
3053 if (drhd->ignored || drhd->include_all)
3054 continue;
3055
3056 for (i = 0; i < drhd->devices_cnt; i++)
3057 if (drhd->devices[i] &&
3058 !IS_GFX_DEVICE(drhd->devices[i]))
3059 break;
3060
3061 if (i < drhd->devices_cnt)
3062 continue;
3063
3064 /* bypass IOMMU if it is just for gfx devices */
3065 drhd->ignored = 1;
3066 for (i = 0; i < drhd->devices_cnt; i++) {
3067 if (!drhd->devices[i])
3068 continue;
358dd8ac 3069 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3070 }
3071 }
3072}
3073
f59c7b69
FY		 3074#ifdef CONFIG_SUSPEND
3075static int init_iommu_hw(void)
3076{
3077 struct dmar_drhd_unit *drhd;
3078 struct intel_iommu *iommu = NULL;
3079
3080 for_each_active_iommu(iommu, drhd)
3081 if (iommu->qi)
3082 dmar_reenable_qi(iommu);
3083
3084 for_each_active_iommu(iommu, drhd) {
3085 iommu_flush_write_buffer(iommu);
3086
3087 iommu_set_root_entry(iommu);
3088
3089 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3090 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3091 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3092 DMA_TLB_GLOBAL_FLUSH);
f59c7b69 3093 iommu_enable_translation(iommu);
b94996c9 3094 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3095 }
3096
3097 return 0;
3098}
3099
3100static void iommu_flush_all(void)
3101{
3102 struct dmar_drhd_unit *drhd;
3103 struct intel_iommu *iommu;
3104
3105 for_each_active_iommu(iommu, drhd) {
3106 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3107 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3108 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3109 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3110 }
3111}
3112
3113static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3114{
3115 struct dmar_drhd_unit *drhd;
3116 struct intel_iommu *iommu = NULL;
3117 unsigned long flag;
3118
3119 for_each_active_iommu(iommu, drhd) {
3120 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3121 GFP_ATOMIC);
3122 if (!iommu->iommu_state)
3123 goto nomem;
3124 }
3125
3126 iommu_flush_all();
3127
3128 for_each_active_iommu(iommu, drhd) {
3129 iommu_disable_translation(iommu);
3130
3131 spin_lock_irqsave(&iommu->register_lock, flag);
3132
3133 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3134 readl(iommu->reg + DMAR_FECTL_REG);
3135 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3136 readl(iommu->reg + DMAR_FEDATA_REG);
3137 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3138 readl(iommu->reg + DMAR_FEADDR_REG);
3139 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3140 readl(iommu->reg + DMAR_FEUADDR_REG);
3141
3142 spin_unlock_irqrestore(&iommu->register_lock, flag);
3143 }
3144 return 0;
3145
3146nomem:
3147 for_each_active_iommu(iommu, drhd)
3148 kfree(iommu->iommu_state);
3149
3150 return -ENOMEM;
3151}
3152
3153static int iommu_resume(struct sys_device *dev)
3154{
3155 struct dmar_drhd_unit *drhd;
3156 struct intel_iommu *iommu = NULL;
3157 unsigned long flag;
3158
3159 if (init_iommu_hw()) {
3160 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3161 return -EIO;
3162 }
3163
3164 for_each_active_iommu(iommu, drhd) {
3165
3166 spin_lock_irqsave(&iommu->register_lock, flag);
3167
3168 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3169 iommu->reg + DMAR_FECTL_REG);
3170 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3171 iommu->reg + DMAR_FEDATA_REG);
3172 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3173 iommu->reg + DMAR_FEADDR_REG);
3174 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3175 iommu->reg + DMAR_FEUADDR_REG);
3176
3177 spin_unlock_irqrestore(&iommu->register_lock, flag);
3178 }
3179
3180 for_each_active_iommu(iommu, drhd)
3181 kfree(iommu->iommu_state);
3182
3183 return 0;
3184}
3185
3186static struct sysdev_class iommu_sysclass = {
3187 .name = "iommu",
3188 .resume = iommu_resume,
3189 .suspend = iommu_suspend,
3190};
3191
3192static struct sys_device device_iommu = {
3193 .cls = &iommu_sysclass,
3194};
3195
3196static int __init init_iommu_sysfs(void)
3197{
3198 int error;
3199
3200 error = sysdev_class_register(&iommu_sysclass);
3201 if (error)
3202 return error;
3203
3204 error = sysdev_register(&device_iommu);
3205 if (error)
3206 sysdev_class_unregister(&iommu_sysclass);
3207
3208 return error;
3209}
3210
3211#else
3212static int __init init_iommu_sysfs(void)
3213{
3214 return 0;
3215}
3216#endif /* CONFIG_PM */
3217
99dcaded
FY		 3218/*
3219 * Here we only respond to action of unbound device from driver.
3220 *
3221 * Added device is not attached to its DMAR domain here yet. That will happen
3222 * when mapping the device to iova.
3223 */
3224static int device_notifier(struct notifier_block *nb,
3225 unsigned long action, void *data)
3226{
3227 struct device *dev = data;
3228 struct pci_dev *pdev = to_pci_dev(dev);
3229 struct dmar_domain *domain;
3230
3231 domain = find_domain(pdev);
3232 if (!domain)
3233 return 0;
3234
3235 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through)
3236 domain_remove_one_dev_info(domain, pdev);
3237
3238 return 0;
3239}
3240
3241static struct notifier_block device_nb = {
3242 .notifier_call = device_notifier,
3243};
3244
ba395927
KA		 3245int __init intel_iommu_init(void)
3246{
3247 int ret = 0;
a59b50e9 3248 int force_on = 0;
ba395927 3249
a59b50e9
JC		 3250 /* VT-d is required for a TXT/tboot launch, so enforce that */
3251 force_on = tboot_force_iommu();
3252
3253 if (dmar_table_init()) {
3254 if (force_on)
3255 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3256 return -ENODEV;
a59b50e9 3257 }
ba395927 3258
a59b50e9
JC		 3259 if (dmar_dev_scope_init()) {
3260 if (force_on)
3261 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3262 return -ENODEV;
a59b50e9 3263 }
1886e8a9 3264
2ae21010
SS		 3265 /*
3266 * Check the need for DMA-remapping initialization now.
3267 * Above initialization will also be used by Interrupt-remapping.
3268 */
75f1cdf1 3269 if (no_iommu || dmar_disabled)
2ae21010
SS		 3270 return -ENODEV;
3271
ba395927
KA		 3272 iommu_init_mempool();
3273 dmar_init_reserved_ranges();
3274
3275 init_no_remapping_devices();
3276
3277 ret = init_dmars();
3278 if (ret) {
a59b50e9
JC		 3279 if (force_on)
3280 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3281 printk(KERN_ERR "IOMMU: dmar init failed\n");
3282 put_iova_domain(&reserved_iova_list);
3283 iommu_exit_mempool();
3284 return ret;
3285 }
3286 printk(KERN_INFO
3287 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3288
5e0d2a6f 3289 init_timer(&unmap_timer);
75f1cdf1
FT		 3290#ifdef CONFIG_SWIOTLB
3291 swiotlb = 0;
3292#endif
19943b0e 3293 dma_ops = &intel_dma_ops;
4ed0d3e6 3294
f59c7b69 3295 init_iommu_sysfs();
a8bcbb0d
JR		 3296
3297 register_iommu(&intel_iommu_ops);
3298
99dcaded
FY		 3299 bus_register_notifier(&pci_bus_type, &device_nb);
3300
ba395927
KA		 3301 return 0;
3302}
e820482c 3303
3199aa6b
HW		 3304static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3305 struct pci_dev *pdev)
3306{
3307 struct pci_dev *tmp, *parent;
3308
3309 if (!iommu || !pdev)
3310 return;
3311
3312 /* dependent device detach */
3313 tmp = pci_find_upstream_pcie_bridge(pdev);
3314 /* Secondary interface's bus number and devfn 0 */
3315 if (tmp) {
3316 parent = pdev->bus->self;
3317 while (parent != tmp) {
3318 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3319 parent->devfn);
3199aa6b
HW		 3320 parent = parent->bus->self;
3321 }
5f4d91a1 3322 if (pci_is_pcie(tmp)) /* this is a PCIE-to-PCI bridge */
3199aa6b
HW		 3323 iommu_detach_dev(iommu,
3324 tmp->subordinate->number, 0);
3325 else /* this is a legacy PCI bridge */
276dbf99
DW		 3326 iommu_detach_dev(iommu, tmp->bus->number,
3327 tmp->devfn);
3199aa6b
HW		 3328 }
3329}
3330
2c2e2c38 3331static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3332 struct pci_dev *pdev)
3333{
3334 struct device_domain_info *info;
3335 struct intel_iommu *iommu;
3336 unsigned long flags;
3337 int found = 0;
3338 struct list_head *entry, *tmp;
3339
276dbf99
DW		 3340 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3341 pdev->devfn);
c7151a8d
WH		 3342 if (!iommu)
3343 return;
3344
3345 spin_lock_irqsave(&device_domain_lock, flags);
3346 list_for_each_safe(entry, tmp, &domain->devices) {
3347 info = list_entry(entry, struct device_domain_info, link);
276dbf99 3348 /* No need to compare PCI domain; it has to be the same */
c7151a8d
WH		 3349 if (info->bus == pdev->bus->number &&
3350 info->devfn == pdev->devfn) {
3351 list_del(&info->link);
3352 list_del(&info->global);
3353 if (info->dev)
3354 info->dev->dev.archdata.iommu = NULL;
3355 spin_unlock_irqrestore(&device_domain_lock, flags);
3356
93a23a72 3357 iommu_disable_dev_iotlb(info);
c7151a8d 3358 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3359 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3360 free_devinfo_mem(info);
3361
3362 spin_lock_irqsave(&device_domain_lock, flags);
3363
3364 if (found)
3365 break;
3366 else
3367 continue;
3368 }
3369
3370 /* if there is no other devices under the same iommu
3371 * owned by this domain, clear this iommu in iommu_bmp
3372 * update iommu count and coherency
3373 */
276dbf99
DW		 3374 if (iommu == device_to_iommu(info->segment, info->bus,
3375 info->devfn))
c7151a8d
WH		 3376 found = 1;
3377 }
3378
3379 if (found == 0) {
3380 unsigned long tmp_flags;
3381 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3382 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3383 domain->iommu_count--;
58c610bd 3384 domain_update_iommu_cap(domain);
c7151a8d
WH		 3385 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3386 }
3387
3388 spin_unlock_irqrestore(&device_domain_lock, flags);
3389}
3390
3391static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3392{
3393 struct device_domain_info *info;
3394 struct intel_iommu *iommu;
3395 unsigned long flags1, flags2;
3396
3397 spin_lock_irqsave(&device_domain_lock, flags1);
3398 while (!list_empty(&domain->devices)) {
3399 info = list_entry(domain->devices.next,
3400 struct device_domain_info, link);
3401 list_del(&info->link);
3402 list_del(&info->global);
3403 if (info->dev)
3404 info->dev->dev.archdata.iommu = NULL;
3405
3406 spin_unlock_irqrestore(&device_domain_lock, flags1);
3407
93a23a72 3408 iommu_disable_dev_iotlb(info);
276dbf99 3409 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3410 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3411 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3412
3413 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3414 * and capabilities
c7151a8d
WH		 3415 */
3416 spin_lock_irqsave(&domain->iommu_lock, flags2);
3417 if (test_and_clear_bit(iommu->seq_id,
3418 &domain->iommu_bmp)) {
3419 domain->iommu_count--;
58c610bd 3420 domain_update_iommu_cap(domain);
c7151a8d
WH		 3421 }
3422 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3423
3424 free_devinfo_mem(info);
3425 spin_lock_irqsave(&device_domain_lock, flags1);
3426 }
3427 spin_unlock_irqrestore(&device_domain_lock, flags1);
3428}
3429
5e98c4b1
WH		 3430/* domain id for virtual machine, it won't be set in context */
3431static unsigned long vm_domid;
3432
fe40f1e0
WH		 3433static int vm_domain_min_agaw(struct dmar_domain *domain)
3434{
3435 int i;
3436 int min_agaw = domain->agaw;
3437
3438 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3439 for (; i < g_num_of_iommus; ) {
3440 if (min_agaw > g_iommus[i]->agaw)
3441 min_agaw = g_iommus[i]->agaw;
3442
3443 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3444 }
3445
3446 return min_agaw;
3447}
3448
5e98c4b1
WH		 3449static struct dmar_domain *iommu_alloc_vm_domain(void)
3450{
3451 struct dmar_domain *domain;
3452
3453 domain = alloc_domain_mem();
3454 if (!domain)
3455 return NULL;
3456
3457 domain->id = vm_domid++;
3458 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3459 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3460
3461 return domain;
3462}
3463
2c2e2c38 3464static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3465{
3466 int adjust_width;
3467
3468 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3469 spin_lock_init(&domain->iommu_lock);
3470
3471 domain_reserve_special_ranges(domain);
3472
3473 /* calculate AGAW */
3474 domain->gaw = guest_width;
3475 adjust_width = guestwidth_to_adjustwidth(guest_width);
3476 domain->agaw = width_to_agaw(adjust_width);
3477
3478 INIT_LIST_HEAD(&domain->devices);
3479
3480 domain->iommu_count = 0;
3481 domain->iommu_coherency = 0;
c5b15255 3482 domain->iommu_snooping = 0;
fe40f1e0 3483 domain->max_addr = 0;
5e98c4b1
WH		 3484
3485 /* always allocate the top pgd */
3486 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
3487 if (!domain->pgd)
3488 return -ENOMEM;
3489 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3490 return 0;
3491}
3492
3493static void iommu_free_vm_domain(struct dmar_domain *domain)
3494{
3495 unsigned long flags;
3496 struct dmar_drhd_unit *drhd;
3497 struct intel_iommu *iommu;
3498 unsigned long i;
3499 unsigned long ndomains;
3500
3501 for_each_drhd_unit(drhd) {
3502 if (drhd->ignored)
3503 continue;
3504 iommu = drhd->iommu;
3505
3506 ndomains = cap_ndoms(iommu->cap);
3507 i = find_first_bit(iommu->domain_ids, ndomains);
3508 for (; i < ndomains; ) {
3509 if (iommu->domains[i] == domain) {
3510 spin_lock_irqsave(&iommu->lock, flags);
3511 clear_bit(i, iommu->domain_ids);
3512 iommu->domains[i] = NULL;
3513 spin_unlock_irqrestore(&iommu->lock, flags);
3514 break;
3515 }
3516 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3517 }
3518 }
3519}
3520
3521static void vm_domain_exit(struct dmar_domain *domain)
3522{
5e98c4b1
WH		 3523 /* Domain 0 is reserved, so dont process it */
3524 if (!domain)
3525 return;
3526
3527 vm_domain_remove_all_dev_info(domain);
3528 /* destroy iovas */
3529 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3530
3531 /* clear ptes */
595badf5 3532 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3533
3534 /* free page tables */
d794dc9b 3535 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3536
3537 iommu_free_vm_domain(domain);
3538 free_domain_mem(domain);
3539}
3540
5d450806 3541static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3542{
5d450806 3543 struct dmar_domain *dmar_domain;
38717946 3544
5d450806
JR		 3545 dmar_domain = iommu_alloc_vm_domain();
3546 if (!dmar_domain) {
38717946 3547 printk(KERN_ERR
5d450806
JR		 3548 "intel_iommu_domain_init: dmar_domain == NULL\n");
3549 return -ENOMEM;
38717946 3550 }
2c2e2c38 3551 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3552 printk(KERN_ERR
5d450806
JR		 3553 "intel_iommu_domain_init() failed\n");
3554 vm_domain_exit(dmar_domain);
3555 return -ENOMEM;
38717946 3556 }
5d450806 3557 domain->priv = dmar_domain;
faa3d6f5 3558
5d450806 3559 return 0;
38717946 3560}
38717946 3561
5d450806 3562static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3563{
5d450806
JR		 3564 struct dmar_domain *dmar_domain = domain->priv;
3565
3566 domain->priv = NULL;
3567 vm_domain_exit(dmar_domain);
38717946 3568}
38717946 3569
4c5478c9
JR		 3570static int intel_iommu_attach_device(struct iommu_domain *domain,
3571 struct device *dev)
38717946 3572{
4c5478c9
JR		 3573 struct dmar_domain *dmar_domain = domain->priv;
3574 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3575 struct intel_iommu *iommu;
3576 int addr_width;
3577 u64 end;
faa3d6f5
WH		 3578
3579 /* normally pdev is not mapped */
3580 if (unlikely(domain_context_mapped(pdev))) {
3581 struct dmar_domain *old_domain;
3582
3583 old_domain = find_domain(pdev);
3584 if (old_domain) {
2c2e2c38
FY		 3585 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3586 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3587 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3588 else
3589 domain_remove_dev_info(old_domain);
3590 }
3591 }
3592
276dbf99
DW		 3593 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3594 pdev->devfn);
fe40f1e0
WH		 3595 if (!iommu)
3596 return -ENODEV;
3597
3598 /* check if this iommu agaw is sufficient for max mapped address */
3599 addr_width = agaw_to_width(iommu->agaw);
3600 end = DOMAIN_MAX_ADDR(addr_width);
3601 end = end & VTD_PAGE_MASK;
4c5478c9 3602 if (end < dmar_domain->max_addr) {
fe40f1e0
WH		 3603 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3604 "sufficient for the mapped address (%llx)\n",
4c5478c9 3605 __func__, iommu->agaw, dmar_domain->max_addr);
fe40f1e0
WH		 3606 return -EFAULT;
3607 }
3608
5fe60f4e 3609 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 3610}
38717946 3611
4c5478c9
JR		 3612static void intel_iommu_detach_device(struct iommu_domain *domain,
3613 struct device *dev)
38717946 3614{
4c5478c9
JR		 3615 struct dmar_domain *dmar_domain = domain->priv;
3616 struct pci_dev *pdev = to_pci_dev(dev);
3617
2c2e2c38 3618 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 3619}
c7151a8d 3620
dde57a21
JR		 3621static int intel_iommu_map_range(struct iommu_domain *domain,
3622 unsigned long iova, phys_addr_t hpa,
3623 size_t size, int iommu_prot)
faa3d6f5 3624{
dde57a21 3625 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 3626 u64 max_addr;
3627 int addr_width;
dde57a21 3628 int prot = 0;
faa3d6f5 3629 int ret;
fe40f1e0 3630
dde57a21
JR		 3631 if (iommu_prot & IOMMU_READ)
3632 prot |= DMA_PTE_READ;
3633 if (iommu_prot & IOMMU_WRITE)
3634 prot |= DMA_PTE_WRITE;
9cf06697
SY		 3635 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3636 prot |= DMA_PTE_SNP;
dde57a21 3637
163cc52c 3638 max_addr = iova + size;
dde57a21 3639 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 3640 int min_agaw;
3641 u64 end;
3642
3643 /* check if minimum agaw is sufficient for mapped address */
dde57a21 3644 min_agaw = vm_domain_min_agaw(dmar_domain);
fe40f1e0
WH		 3645 addr_width = agaw_to_width(min_agaw);
3646 end = DOMAIN_MAX_ADDR(addr_width);
3647 end = end & VTD_PAGE_MASK;
3648 if (end < max_addr) {
3649 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3650 "sufficient for the mapped address (%llx)\n",
3651 __func__, min_agaw, max_addr);
3652 return -EFAULT;
3653 }
dde57a21 3654 dmar_domain->max_addr = max_addr;
fe40f1e0 3655 }
ad051221
DW		 3656 /* Round up size to next multiple of PAGE_SIZE, if it and
3657 the low bits of hpa would take us onto the next page */
88cb6a74 3658 size = aligned_nrpages(hpa, size);
ad051221
DW		 3659 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3660 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 3661 return ret;
38717946 3662}
38717946 3663
dde57a21
JR		 3664static void intel_iommu_unmap_range(struct iommu_domain *domain,
3665 unsigned long iova, size_t size)
38717946 3666{
dde57a21 3667 struct dmar_domain *dmar_domain = domain->priv;
faa3d6f5 3668
4b99d352
SY		 3669 if (!size)
3670 return;
3671
163cc52c
DW		 3672 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3673 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 3674
163cc52c
DW		 3675 if (dmar_domain->max_addr == iova + size)
3676 dmar_domain->max_addr = iova;
38717946 3677}
38717946 3678
d14d6577
JR		 3679static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3680 unsigned long iova)
38717946 3681{
d14d6577 3682 struct dmar_domain *dmar_domain = domain->priv;
38717946 3683 struct dma_pte *pte;
faa3d6f5 3684 u64 phys = 0;
38717946 3685
b026fd28 3686 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
38717946 3687 if (pte)
faa3d6f5 3688 phys = dma_pte_addr(pte);
38717946 3689
faa3d6f5 3690 return phys;
38717946 3691}
a8bcbb0d 3692
dbb9fd86
SY		 3693static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3694 unsigned long cap)
3695{
3696 struct dmar_domain *dmar_domain = domain->priv;
3697
3698 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3699 return dmar_domain->iommu_snooping;
3700
3701 return 0;
3702}
3703
a8bcbb0d
JR		 3704static struct iommu_ops intel_iommu_ops = {
3705 .domain_init = intel_iommu_domain_init,
3706 .domain_destroy = intel_iommu_domain_destroy,
3707 .attach_dev = intel_iommu_attach_device,
3708 .detach_dev = intel_iommu_detach_device,
3709 .map = intel_iommu_map_range,
3710 .unmap = intel_iommu_unmap_range,
3711 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 3712 .domain_has_cap = intel_iommu_domain_has_cap,
a8bcbb0d 3713};
9af88143
DW		 3714
3715static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3716{
3717 /*
3718 * Mobile 4 Series Chipset neglects to set RWBF capability,
3719 * but needs it:
3720 */
3721 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3722 rwbf_quirk = 1;
3723}
3724
3725DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
e0fc7e0b
DW		 3726
3727/* On Tylersburg chipsets, some BIOSes have been known to enable the
3728 ISOCH DMAR unit for the Azalia sound device, but not give it any
3729 TLB entries, which causes it to deadlock. Check for that. We do
3730 this in a function called from init_dmars(), instead of in a PCI
3731 quirk, because we don't want to print the obnoxious "BIOS broken"
3732 message if VT-d is actually disabled.
3733*/
3734static void __init check_tylersburg_isoch(void)
3735{
3736 struct pci_dev *pdev;
3737 uint32_t vtisochctrl;
3738
3739 /* If there's no Azalia in the system anyway, forget it. */
3740 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3741 if (!pdev)
3742 return;
3743 pci_dev_put(pdev);
3744
3745 /* System Management Registers. Might be hidden, in which case
3746 we can't do the sanity check. But that's OK, because the
3747 known-broken BIOSes _don't_ actually hide it, so far. */
3748 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3749 if (!pdev)
3750 return;
3751
3752 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3753 pci_dev_put(pdev);
3754 return;
3755 }
3756
3757 pci_dev_put(pdev);
3758
3759 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3760 if (vtisochctrl & 1)
3761 return;
3762
3763 /* Drop all bits other than the number of TLB entries */
3764 vtisochctrl &= 0x1c;
3765
3766 /* If we have the recommended number of TLB entries (16), fine. */
3767 if (vtisochctrl == 0x10)
3768 return;
3769
3770 /* Zero TLB entries? You get to ride the short bus to school. */
3771 if (!vtisochctrl) {
3772 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3773 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3774 dmi_get_system_info(DMI_BIOS_VENDOR),
3775 dmi_get_system_info(DMI_BIOS_VERSION),
3776 dmi_get_system_info(DMI_PRODUCT_VERSION));
3777 iommu_identity_mapping |= IDENTMAP_AZALIA;
3778 return;
3779 }
3780
3781 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3782 vtisochctrl);
3783}
kernel source for telekom puls (alcatel/mediatek)