projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'v3.10.103' into update
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
54485c30 27#include <linux/export.h>
ba395927
KA		 28#include <linux/slab.h>
29#include <linux/irq.h>
30#include <linux/interrupt.h>
ba395927
KA		 31#include <linux/spinlock.h>
32#include <linux/pci.h>
33#include <linux/dmar.h>
34#include <linux/dma-mapping.h>
35#include <linux/mempool.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
8a8f422d 45#include <asm/irq_remapping.h>
ba395927 46#include <asm/cacheflush.h>
46a7fa27 47#include <asm/iommu.h>
ba395927 48
078e1ee2 49#include "irq_remapping.h"
61e015ac 50#include "pci.h"
078e1ee2 51
5b6985ce
FY		 52#define ROOT_SIZE VTD_PAGE_SIZE
53#define CONTEXT_SIZE VTD_PAGE_SIZE
54
ba395927
KA		 55#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
56#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 57#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 58
59#define IOAPIC_RANGE_START (0xfee00000)
60#define IOAPIC_RANGE_END (0xfeefffff)
61#define IOVA_START_ADDR (0x1000)
62
63#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
64
4ed0d3e6
FY		 65#define MAX_AGAW_WIDTH 64
66
2ebe3151
DW		 67#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
68#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
69
70/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
71 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
72#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
73 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
74#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 75
f27be03b 76#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 77#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 78#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 79
df08cdc7
AM		 80/* page table handling */
81#define LEVEL_STRIDE (9)
82#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
83
6d1c56a9
OBC		 84/*
85 * This bitmap is used to advertise the page sizes our hardware support
86 * to the IOMMU core, which will then use this information to split
87 * physically contiguous memory regions it is mapping into page sizes
88 * that we support.
89 *
90 * Traditionally the IOMMU core just handed us the mappings directly,
91 * after making sure the size is an order of a 4KiB page and that the
92 * mapping has natural alignment.
93 *
94 * To retain this behavior, we currently advertise that we support
95 * all page sizes that are an order of 4KiB.
96 *
97 * If at some point we'd like to utilize the IOMMU core's new behavior,
98 * we could change this to advertise the real page sizes we support.
99 */
100#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
101
df08cdc7
AM		 102static inline int agaw_to_level(int agaw)
103{
104 return agaw + 2;
105}
106
107static inline int agaw_to_width(int agaw)
108{
109 return 30 + agaw * LEVEL_STRIDE;
110}
111
112static inline int width_to_agaw(int width)
113{
114 return (width - 30) / LEVEL_STRIDE;
115}
116
117static inline unsigned int level_to_offset_bits(int level)
118{
119 return (level - 1) * LEVEL_STRIDE;
120}
121
122static inline int pfn_level_offset(unsigned long pfn, int level)
123{
124 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
125}
126
127static inline unsigned long level_mask(int level)
128{
129 return -1UL << level_to_offset_bits(level);
130}
131
132static inline unsigned long level_size(int level)
133{
134 return 1UL << level_to_offset_bits(level);
135}
136
137static inline unsigned long align_to_level(unsigned long pfn, int level)
138{
139 return (pfn + level_size(level) - 1) & level_mask(level);
140}
fd18de50 141
6dd9a7c7
YS		 142static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
143{
144 return 1 << ((lvl - 1) * LEVEL_STRIDE);
145}
146
dd4e8319
DW		 147/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
148 are never going to work. */
149static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
150{
151 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
152}
153
154static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
155{
156 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
157}
158static inline unsigned long page_to_dma_pfn(struct page *pg)
159{
160 return mm_to_dma_pfn(page_to_pfn(pg));
161}
162static inline unsigned long virt_to_dma_pfn(void *p)
163{
164 return page_to_dma_pfn(virt_to_page(p));
165}
166
d9630fe9
WH		 167/* global iommu list, set NULL for ignored DMAR units */
168static struct intel_iommu **g_iommus;
169
e0fc7e0b 170static void __init check_tylersburg_isoch(void);
9af88143
DW		 171static int rwbf_quirk;
172
b779260b
JC		 173/*
174 * set to 1 to panic kernel if can't successfully enable VT-d
175 * (used when kernel is launched w/ TXT)
176 */
177static int force_on = 0;
178
46b08e1a
MM		 179/*
180 * 0: Present
181 * 1-11: Reserved
182 * 12-63: Context Ptr (12 - (haw-1))
183 * 64-127: Reserved
184 */
185struct root_entry {
186 u64 val;
187 u64 rsvd1;
188};
189#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
190static inline bool root_present(struct root_entry *root)
191{
192 return (root->val & 1);
193}
194static inline void set_root_present(struct root_entry *root)
195{
196 root->val |= 1;
197}
198static inline void set_root_value(struct root_entry *root, unsigned long value)
199{
200 root->val |= value & VTD_PAGE_MASK;
201}
202
203static inline struct context_entry *
204get_context_addr_from_root(struct root_entry *root)
205{
206 return (struct context_entry *)
207 (root_present(root)?phys_to_virt(
208 root->val & VTD_PAGE_MASK) :
209 NULL);
210}
211
7a8fc25e
MM		 212/*
213 * low 64 bits:
214 * 0: present
215 * 1: fault processing disable
216 * 2-3: translation type
217 * 12-63: address space root
218 * high 64 bits:
219 * 0-2: address width
220 * 3-6: aval
221 * 8-23: domain id
222 */
223struct context_entry {
224 u64 lo;
225 u64 hi;
226};
c07e7d21
MM		 227
228static inline bool context_present(struct context_entry *context)
229{
230 return (context->lo & 1);
231}
232static inline void context_set_present(struct context_entry *context)
233{
234 context->lo |= 1;
235}
236
237static inline void context_set_fault_enable(struct context_entry *context)
238{
239 context->lo &= (((u64)-1) << 2) | 1;
240}
241
c07e7d21
MM		 242static inline void context_set_translation_type(struct context_entry *context,
243 unsigned long value)
244{
245 context->lo &= (((u64)-1) << 4) | 3;
246 context->lo |= (value & 3) << 2;
247}
248
249static inline void context_set_address_root(struct context_entry *context,
250 unsigned long value)
251{
252 context->lo |= value & VTD_PAGE_MASK;
253}
254
255static inline void context_set_address_width(struct context_entry *context,
256 unsigned long value)
257{
258 context->hi |= value & 7;
259}
260
261static inline void context_set_domain_id(struct context_entry *context,
262 unsigned long value)
263{
264 context->hi |= (value & ((1 << 16) - 1)) << 8;
265}
266
267static inline void context_clear_entry(struct context_entry *context)
268{
269 context->lo = 0;
270 context->hi = 0;
271}
7a8fc25e 272
622ba12a
MM		 273/*
274 * 0: readable
275 * 1: writable
276 * 2-6: reserved
277 * 7: super page
9cf06697
SY		 278 * 8-10: available
279 * 11: snoop behavior
622ba12a
MM		 280 * 12-63: Host physcial address
281 */
282struct dma_pte {
283 u64 val;
284};
622ba12a 285
19c239ce
MM		 286static inline void dma_clear_pte(struct dma_pte *pte)
287{
288 pte->val = 0;
289}
290
291static inline void dma_set_pte_readable(struct dma_pte *pte)
292{
293 pte->val |= DMA_PTE_READ;
294}
295
296static inline void dma_set_pte_writable(struct dma_pte *pte)
297{
298 pte->val |= DMA_PTE_WRITE;
299}
300
9cf06697
SY		 301static inline void dma_set_pte_snp(struct dma_pte *pte)
302{
303 pte->val |= DMA_PTE_SNP;
304}
305
19c239ce
MM		 306static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
307{
308 pte->val = (pte->val & ~3) | (prot & 3);
309}
310
311static inline u64 dma_pte_addr(struct dma_pte *pte)
312{
c85994e4
DW		 313#ifdef CONFIG_64BIT
314 return pte->val & VTD_PAGE_MASK;
315#else
316 /* Must have a full atomic 64-bit read */
1a8bd481 317 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 318#endif
19c239ce
MM		 319}
320
dd4e8319 321static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 322{
dd4e8319 323 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 324}
325
326static inline bool dma_pte_present(struct dma_pte *pte)
327{
328 return (pte->val & 3) != 0;
329}
622ba12a 330
4399c8bf
AK		 331static inline bool dma_pte_superpage(struct dma_pte *pte)
332{
333 return (pte->val & (1 << 7));
334}
335
75e6bf96
DW		 336static inline int first_pte_in_page(struct dma_pte *pte)
337{
338 return !((unsigned long)pte & ~VTD_PAGE_MASK);
339}
340
2c2e2c38
FY		 341/*
342 * This domain is a statically identity mapping domain.
343 * 1. This domain creats a static 1:1 mapping to all usable memory.
344 * 2. It maps to each iommu if successful.
345 * 3. Each iommu mapps to this domain if successful.
346 */
19943b0e
DW		 347static struct dmar_domain *si_domain;
348static int hw_pass_through = 1;
2c2e2c38 349
3b5410e7 350/* devices under the same p2p bridge are owned in one domain */
cdc7b837 351#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 352
1ce28feb
WH		 353/* domain represents a virtual machine, more than one devices
354 * across iommus may be owned in one domain, e.g. kvm guest.
355 */
356#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
357
2c2e2c38
FY		 358/* si_domain contains mulitple devices */
359#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
360
1b198bb0
MT		 361/* define the limit of IOMMUs supported in each domain */
362#ifdef CONFIG_X86
363# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
364#else
365# define IOMMU_UNITS_SUPPORTED 64
366#endif
367
99126f7c
MM		 368struct dmar_domain {
369 int id; /* domain id */
4c923d47 370 int nid; /* node id */
1b198bb0
MT		 371 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
372 /* bitmap of iommus this domain uses*/
99126f7c
MM		 373
374 struct list_head devices; /* all devices' list */
375 struct iova_domain iovad; /* iova's that belong to this domain */
376
377 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 378 int gaw; /* max guest address width */
379
380 /* adjusted guest address width, 0 is level 2 30-bit */
381 int agaw;
382
3b5410e7 383 int flags; /* flags to find out type of domain */
8e604097
WH		 384
385 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 386 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 387 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 388 int iommu_superpage;/* Level of superpages supported:
389 0 == 4KiB (no superpages), 1 == 2MiB,
390 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 391 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 392 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 393};
394
a647dacb
MM		 395/* PCI domain-device relationship */
396struct device_domain_info {
397 struct list_head link; /* link to domain siblings */
398 struct list_head global; /* link to global list */
276dbf99
DW		 399 int segment; /* PCI domain */
400 u8 bus; /* PCI bus number */
a647dacb 401 u8 devfn; /* PCI devfn number */
45e829ea 402 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 403 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 404 struct dmar_domain *domain; /* pointer to domain */
405};
406
5e0d2a6f 407static void flush_unmaps_timeout(unsigned long data);
408
409DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
410
80b20dd8 411#define HIGH_WATER_MARK 250
412struct deferred_flush_tables {
413 int next;
414 struct iova *iova[HIGH_WATER_MARK];
415 struct dmar_domain *domain[HIGH_WATER_MARK];
416};
417
418static struct deferred_flush_tables *deferred_flush;
419
5e0d2a6f 420/* bitmap for indexing intel_iommus */
5e0d2a6f 421static int g_num_of_iommus;
422
423static DEFINE_SPINLOCK(async_umap_flush_lock);
424static LIST_HEAD(unmaps_to_do);
425
426static int timer_on;
427static long list_size;
5e0d2a6f 428
ba395927
KA		 429static void domain_remove_dev_info(struct dmar_domain *domain);
430
d3f13810 431#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 432int dmar_disabled = 0;
433#else
434int dmar_disabled = 1;
d3f13810 435#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 436
8bc1f85c
ED		 437int intel_iommu_enabled = 0;
438EXPORT_SYMBOL_GPL(intel_iommu_enabled);
439
2d9e667e 440static int dmar_map_gfx = 1;
7d3b03ce 441static int dmar_forcedac;
5e0d2a6f 442static int intel_iommu_strict;
6dd9a7c7 443static int intel_iommu_superpage = 1;
ba395927 444
c0771df8
DW		 445int intel_iommu_gfx_mapped;
446EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
447
ba395927
KA		 448#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
449static DEFINE_SPINLOCK(device_domain_lock);
450static LIST_HEAD(device_domain_list);
451
a8bcbb0d
JR		 452static struct iommu_ops intel_iommu_ops;
453
ba395927
KA		 454static int __init intel_iommu_setup(char *str)
455{
456 if (!str)
457 return -EINVAL;
458 while (*str) {
0cd5c3c8
KM		 459 if (!strncmp(str, "on", 2)) {
460 dmar_disabled = 0;
461 printk(KERN_INFO "Intel-IOMMU: enabled\n");
462 } else if (!strncmp(str, "off", 3)) {
ba395927 463 dmar_disabled = 1;
0cd5c3c8 464 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 465 } else if (!strncmp(str, "igfx_off", 8)) {
466 dmar_map_gfx = 0;
467 printk(KERN_INFO
468 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 469 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 470 printk(KERN_INFO
7d3b03ce
KA		 471 "Intel-IOMMU: Forcing DAC for PCI devices\n");
472 dmar_forcedac = 1;
5e0d2a6f 473 } else if (!strncmp(str, "strict", 6)) {
474 printk(KERN_INFO
475 "Intel-IOMMU: disable batched IOTLB flush\n");
476 intel_iommu_strict = 1;
6dd9a7c7
YS		 477 } else if (!strncmp(str, "sp_off", 6)) {
478 printk(KERN_INFO
479 "Intel-IOMMU: disable supported super page\n");
480 intel_iommu_superpage = 0;
ba395927
KA		 481 }
482
483 str += strcspn(str, ",");
484 while (*str == ',')
485 str++;
486 }
487 return 0;
488}
489__setup("intel_iommu=", intel_iommu_setup);
490
491static struct kmem_cache *iommu_domain_cache;
492static struct kmem_cache *iommu_devinfo_cache;
493static struct kmem_cache *iommu_iova_cache;
494
4c923d47 495static inline void *alloc_pgtable_page(int node)
eb3fa7cb 496{
4c923d47
SS		 497 struct page *page;
498 void *vaddr = NULL;
eb3fa7cb 499
4c923d47
SS		 500 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
501 if (page)
502 vaddr = page_address(page);
eb3fa7cb 503 return vaddr;
ba395927
KA		 504}
505
506static inline void free_pgtable_page(void *vaddr)
507{
508 free_page((unsigned long)vaddr);
509}
510
511static inline void *alloc_domain_mem(void)
512{
354bb65e 513 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 514}
515
38717946 516static void free_domain_mem(void *vaddr)
ba395927
KA		 517{
518 kmem_cache_free(iommu_domain_cache, vaddr);
519}
520
521static inline void * alloc_devinfo_mem(void)
522{
354bb65e 523 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 524}
525
526static inline void free_devinfo_mem(void *vaddr)
527{
528 kmem_cache_free(iommu_devinfo_cache, vaddr);
529}
530
531struct iova *alloc_iova_mem(void)
532{
354bb65e 533 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 534}
535
536void free_iova_mem(struct iova *iova)
537{
538 kmem_cache_free(iommu_iova_cache, iova);
539}
540
1b573683 541
4ed0d3e6 542static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 543{
544 unsigned long sagaw;
545 int agaw = -1;
546
547 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 548 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 549 agaw >= 0; agaw--) {
550 if (test_bit(agaw, &sagaw))
551 break;
552 }
553
554 return agaw;
555}
556
4ed0d3e6
FY		 557/*
558 * Calculate max SAGAW for each iommu.
559 */
560int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
561{
562 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
563}
564
565/*
566 * calculate agaw for each iommu.
567 * "SAGAW" may be different across iommus, use a default agaw, and
568 * get a supported less agaw for iommus that don't support the default agaw.
569 */
570int iommu_calculate_agaw(struct intel_iommu *iommu)
571{
572 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
573}
574
2c2e2c38 575/* This functionin only returns single iommu in a domain */
8c11e798
WH		 576static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
577{
578 int iommu_id;
579
2c2e2c38 580 /* si_domain and vm domain should not get here. */
1ce28feb 581 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 582 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 583
1b198bb0 584 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 585 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
586 return NULL;
587
588 return g_iommus[iommu_id];
589}
590
8e604097
WH		 591static void domain_update_iommu_coherency(struct dmar_domain *domain)
592{
593 int i;
594
2e12bc29
AW		 595 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
596
597 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
8e604097 598
1b198bb0 599 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 600 if (!ecap_coherent(g_iommus[i]->ecap)) {
601 domain->iommu_coherency = 0;
602 break;
603 }
8e604097
WH		 604 }
605}
606
58c610bd
SY		 607static void domain_update_iommu_snooping(struct dmar_domain *domain)
608{
609 int i;
610
611 domain->iommu_snooping = 1;
612
1b198bb0 613 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 614 if (!ecap_sc_support(g_iommus[i]->ecap)) {
615 domain->iommu_snooping = 0;
616 break;
617 }
58c610bd
SY		 618 }
619}
620
6dd9a7c7
YS		 621static void domain_update_iommu_superpage(struct dmar_domain *domain)
622{
8140a95d
AK		 623 struct dmar_drhd_unit *drhd;
624 struct intel_iommu *iommu = NULL;
625 int mask = 0xf;
6dd9a7c7
YS		 626
627 if (!intel_iommu_superpage) {
628 domain->iommu_superpage = 0;
629 return;
630 }
631
8140a95d
AK		 632 /* set iommu_superpage to the smallest common denominator */
633 for_each_active_iommu(iommu, drhd) {
634 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 635 if (!mask) {
636 break;
637 }
638 }
639 domain->iommu_superpage = fls(mask);
640}
641
58c610bd
SY		 642/* Some capabilities may be different across iommus */
643static void domain_update_iommu_cap(struct dmar_domain *domain)
644{
645 domain_update_iommu_coherency(domain);
646 domain_update_iommu_snooping(domain);
6dd9a7c7 647 domain_update_iommu_superpage(domain);
58c610bd
SY		 648}
649
276dbf99 650static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 651{
652 struct dmar_drhd_unit *drhd = NULL;
653 int i;
654
655 for_each_drhd_unit(drhd) {
656 if (drhd->ignored)
657 continue;
276dbf99
DW		 658 if (segment != drhd->segment)
659 continue;
c7151a8d 660
924b6231 661 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 662 if (drhd->devices[i] &&
663 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 664 drhd->devices[i]->devfn == devfn)
665 return drhd->iommu;
4958c5dc
DW		 666 if (drhd->devices[i] &&
667 drhd->devices[i]->subordinate &&
924b6231 668 drhd->devices[i]->subordinate->number <= bus &&
b918c62e 669 drhd->devices[i]->subordinate->busn_res.end >= bus)
924b6231
DW		 670 return drhd->iommu;
671 }
c7151a8d
WH		 672
673 if (drhd->include_all)
674 return drhd->iommu;
675 }
676
677 return NULL;
678}
679
5331fe6f
WH		 680static void domain_flush_cache(struct dmar_domain *domain,
681 void *addr, int size)
682{
683 if (!domain->iommu_coherency)
684 clflush_cache_range(addr, size);
685}
686
ba395927
KA		 687/* Gets context entry for a given bus and devfn */
688static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
689 u8 bus, u8 devfn)
690{
691 struct root_entry *root;
692 struct context_entry *context;
693 unsigned long phy_addr;
694 unsigned long flags;
695
696 spin_lock_irqsave(&iommu->lock, flags);
697 root = &iommu->root_entry[bus];
698 context = get_context_addr_from_root(root);
699 if (!context) {
4c923d47
SS		 700 context = (struct context_entry *)
701 alloc_pgtable_page(iommu->node);
ba395927
KA		 702 if (!context) {
703 spin_unlock_irqrestore(&iommu->lock, flags);
704 return NULL;
705 }
5b6985ce 706 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 707 phy_addr = virt_to_phys((void *)context);
708 set_root_value(root, phy_addr);
709 set_root_present(root);
710 __iommu_flush_cache(iommu, root, sizeof(*root));
711 }
712 spin_unlock_irqrestore(&iommu->lock, flags);
713 return &context[devfn];
714}
715
716static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
717{
718 struct root_entry *root;
719 struct context_entry *context;
720 int ret;
721 unsigned long flags;
722
723 spin_lock_irqsave(&iommu->lock, flags);
724 root = &iommu->root_entry[bus];
725 context = get_context_addr_from_root(root);
726 if (!context) {
727 ret = 0;
728 goto out;
729 }
c07e7d21 730 ret = context_present(&context[devfn]);
ba395927
KA		 731out:
732 spin_unlock_irqrestore(&iommu->lock, flags);
733 return ret;
734}
735
736static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
737{
738 struct root_entry *root;
739 struct context_entry *context;
740 unsigned long flags;
741
742 spin_lock_irqsave(&iommu->lock, flags);
743 root = &iommu->root_entry[bus];
744 context = get_context_addr_from_root(root);
745 if (context) {
c07e7d21 746 context_clear_entry(&context[devfn]);
ba395927
KA		 747 __iommu_flush_cache(iommu, &context[devfn], \
748 sizeof(*context));
749 }
750 spin_unlock_irqrestore(&iommu->lock, flags);
751}
752
753static void free_context_table(struct intel_iommu *iommu)
754{
755 struct root_entry *root;
756 int i;
757 unsigned long flags;
758 struct context_entry *context;
759
760 spin_lock_irqsave(&iommu->lock, flags);
761 if (!iommu->root_entry) {
762 goto out;
763 }
764 for (i = 0; i < ROOT_ENTRY_NR; i++) {
765 root = &iommu->root_entry[i];
766 context = get_context_addr_from_root(root);
767 if (context)
768 free_pgtable_page(context);
769 }
770 free_pgtable_page(iommu->root_entry);
771 iommu->root_entry = NULL;
772out:
773 spin_unlock_irqrestore(&iommu->lock, flags);
774}
775
b026fd28 776static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
4399c8bf 777 unsigned long pfn, int target_level)
ba395927 778{
b026fd28 779 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 780 struct dma_pte *parent, *pte = NULL;
781 int level = agaw_to_level(domain->agaw);
4399c8bf 782 int offset;
ba395927
KA		 783
784 BUG_ON(!domain->pgd);
3de762b3
JS		 785
786 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
787 /* Address beyond IOMMU's addressing capabilities. */
788 return NULL;
789
ba395927
KA		 790 parent = domain->pgd;
791
ba395927
KA		 792 while (level > 0) {
793 void *tmp_page;
794
b026fd28 795 offset = pfn_level_offset(pfn, level);
ba395927 796 pte = &parent[offset];
4399c8bf 797 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7
YS		 798 break;
799 if (level == target_level)
ba395927
KA		 800 break;
801
19c239ce 802 if (!dma_pte_present(pte)) {
c85994e4
DW		 803 uint64_t pteval;
804
4c923d47 805 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 806
206a73c1 807 if (!tmp_page)
ba395927 808 return NULL;
206a73c1 809
c85994e4 810 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 811 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 812 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
813 /* Someone else set it while we were thinking; use theirs. */
814 free_pgtable_page(tmp_page);
815 } else {
816 dma_pte_addr(pte);
817 domain_flush_cache(domain, pte, sizeof(*pte));
818 }
ba395927 819 }
19c239ce 820 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 821 level--;
822 }
823
ba395927
KA		 824 return pte;
825}
826
6dd9a7c7 827
ba395927 828/* return address's pte at specific level */
90dcfb5e
DW		 829static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
830 unsigned long pfn,
6dd9a7c7 831 int level, int *large_page)
ba395927
KA		 832{
833 struct dma_pte *parent, *pte = NULL;
834 int total = agaw_to_level(domain->agaw);
835 int offset;
836
837 parent = domain->pgd;
838 while (level <= total) {
90dcfb5e 839 offset = pfn_level_offset(pfn, total);
ba395927
KA		 840 pte = &parent[offset];
841 if (level == total)
842 return pte;
843
6dd9a7c7
YS		 844 if (!dma_pte_present(pte)) {
845 *large_page = total;
ba395927 846 break;
6dd9a7c7
YS		 847 }
848
849 if (pte->val & DMA_PTE_LARGE_PAGE) {
850 *large_page = total;
851 return pte;
852 }
853
19c239ce 854 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 855 total--;
856 }
857 return NULL;
858}
859
ba395927 860/* clear last level pte, a tlb flush should be followed */
292827cb 861static int dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 862 unsigned long start_pfn,
863 unsigned long last_pfn)
ba395927 864{
04b18e65 865 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 866 unsigned int large_page = 1;
310a5ab9 867 struct dma_pte *first_pte, *pte;
292827cb 868 int order;
66eae846 869
04b18e65 870 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 871 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 872 BUG_ON(start_pfn > last_pfn);
ba395927 873
04b18e65 874 /* we don't need lock here; nobody else touches the iova range */
59c36286 875 do {
6dd9a7c7
YS		 876 large_page = 1;
877 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 878 if (!pte) {
6dd9a7c7 879 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 880 continue;
881 }
6dd9a7c7 882 do {
310a5ab9 883 dma_clear_pte(pte);
6dd9a7c7 884 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 885 pte++;
75e6bf96
DW		 886 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
887
310a5ab9
DW		 888 domain_flush_cache(domain, first_pte,
889 (void *)pte - (void *)first_pte);
59c36286
DW		 890
891 } while (start_pfn && start_pfn <= last_pfn);
292827cb
AK		 892
893 order = (large_page - 1) * 9;
894 return order;
ba395927
KA		 895}
896
1b24e0e4
AW		 897static void dma_pte_free_level(struct dmar_domain *domain, int level,
898 struct dma_pte *pte, unsigned long pfn,
899 unsigned long start_pfn, unsigned long last_pfn)
900{
901 pfn = max(start_pfn, pfn);
902 pte = &pte[pfn_level_offset(pfn, level)];
903
904 do {
905 unsigned long level_pfn;
906 struct dma_pte *level_pte;
907
908 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
909 goto next;
910
911 level_pfn = pfn & level_mask(level - 1);
912 level_pte = phys_to_virt(dma_pte_addr(pte));
913
914 if (level > 2)
915 dma_pte_free_level(domain, level - 1, level_pte,
916 level_pfn, start_pfn, last_pfn);
917
918 /* If range covers entire pagetable, free it */
919 if (!(start_pfn > level_pfn ||
32df365d 920 last_pfn < level_pfn + level_size(level) - 1)) {
1b24e0e4
AW		 921 dma_clear_pte(pte);
922 domain_flush_cache(domain, pte, sizeof(*pte));
923 free_pgtable_page(level_pte);
924 }
925next:
926 pfn += level_size(level);
927 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
928}
929
ba395927
KA		 930/* free page table pages. last level pte should already be cleared */
931static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 932 unsigned long start_pfn,
933 unsigned long last_pfn)
ba395927 934{
6660c63a 935 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 936
6660c63a
DW		 937 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
938 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 939 BUG_ON(start_pfn > last_pfn);
ba395927 940
f3a0a52f 941 /* We don't need lock here; nobody else touches the iova range */
1b24e0e4
AW		 942 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
943 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 944
ba395927 945 /* free pgd */
d794dc9b 946 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 947 free_pgtable_page(domain->pgd);
948 domain->pgd = NULL;
949 }
950}
951
952/* iommu handling */
953static int iommu_alloc_root_entry(struct intel_iommu *iommu)
954{
955 struct root_entry *root;
956 unsigned long flags;
957
4c923d47 958 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 959 if (!root)
960 return -ENOMEM;
961
5b6985ce 962 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 963
964 spin_lock_irqsave(&iommu->lock, flags);
965 iommu->root_entry = root;
966 spin_unlock_irqrestore(&iommu->lock, flags);
967
968 return 0;
969}
970
ba395927
KA		 971static void iommu_set_root_entry(struct intel_iommu *iommu)
972{
973 void *addr;
c416daa9 974 u32 sts;
ba395927
KA		 975 unsigned long flag;
976
977 addr = iommu->root_entry;
978
1f5b3c3f 979 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 980 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
981
c416daa9 982 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 983
984 /* Make sure hardware complete it */
985 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 986 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 987
1f5b3c3f 988 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 989}
990
991static void iommu_flush_write_buffer(struct intel_iommu *iommu)
992{
993 u32 val;
994 unsigned long flag;
995
9af88143 996 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 997 return;
ba395927 998
1f5b3c3f 999 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1000 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1001
1002 /* Make sure hardware complete it */
1003 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1004 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1005
1f5b3c3f 1006 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1007}
1008
1009/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1010static void __iommu_flush_context(struct intel_iommu *iommu,
1011 u16 did, u16 source_id, u8 function_mask,
1012 u64 type)
ba395927
KA		 1013{
1014 u64 val = 0;
1015 unsigned long flag;
1016
ba395927
KA		 1017 switch (type) {
1018 case DMA_CCMD_GLOBAL_INVL:
1019 val = DMA_CCMD_GLOBAL_INVL;
1020 break;
1021 case DMA_CCMD_DOMAIN_INVL:
1022 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1023 break;
1024 case DMA_CCMD_DEVICE_INVL:
1025 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1026 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1027 break;
1028 default:
1029 BUG();
1030 }
1031 val |= DMA_CCMD_ICC;
1032
1f5b3c3f 1033 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1034 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1035
1036 /* Make sure hardware complete it */
1037 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1038 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1039
1f5b3c3f 1040 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1041}
1042
ba395927 1043/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1044static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1045 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1046{
1047 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1048 u64 val = 0, val_iva = 0;
1049 unsigned long flag;
1050
ba395927
KA		 1051 switch (type) {
1052 case DMA_TLB_GLOBAL_FLUSH:
1053 /* global flush doesn't need set IVA_REG */
1054 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1055 break;
1056 case DMA_TLB_DSI_FLUSH:
1057 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1058 break;
1059 case DMA_TLB_PSI_FLUSH:
1060 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1061 /* Note: always flush non-leaf currently */
1062 val_iva = size_order | addr;
1063 break;
1064 default:
1065 BUG();
1066 }
1067 /* Note: set drain read/write */
1068#if 0
1069 /*
1070 * This is probably to be super secure.. Looks like we can
1071 * ignore it without any impact.
1072 */
1073 if (cap_read_drain(iommu->cap))
1074 val |= DMA_TLB_READ_DRAIN;
1075#endif
1076 if (cap_write_drain(iommu->cap))
1077 val |= DMA_TLB_WRITE_DRAIN;
1078
1f5b3c3f 1079 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1080 /* Note: Only uses first TLB reg currently */
1081 if (val_iva)
1082 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1083 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1084
1085 /* Make sure hardware complete it */
1086 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1087 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1088
1f5b3c3f 1089 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1090
1091 /* check IOTLB invalidation granularity */
1092 if (DMA_TLB_IAIG(val) == 0)
1093 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1094 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1095 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1096 (unsigned long long)DMA_TLB_IIRG(type),
1097 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1098}
1099
93a23a72
YZ		 1100static struct device_domain_info *iommu_support_dev_iotlb(
1101 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1102{
1103 int found = 0;
1104 unsigned long flags;
1105 struct device_domain_info *info;
1106 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1107
1108 if (!ecap_dev_iotlb_support(iommu->ecap))
1109 return NULL;
1110
1111 if (!iommu->qi)
1112 return NULL;
1113
1114 spin_lock_irqsave(&device_domain_lock, flags);
1115 list_for_each_entry(info, &domain->devices, link)
1116 if (info->bus == bus && info->devfn == devfn) {
1117 found = 1;
1118 break;
1119 }
1120 spin_unlock_irqrestore(&device_domain_lock, flags);
1121
1122 if (!found || !info->dev)
1123 return NULL;
1124
1125 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1126 return NULL;
1127
1128 if (!dmar_find_matched_atsr_unit(info->dev))
1129 return NULL;
1130
1131 info->iommu = iommu;
1132
1133 return info;
1134}
1135
1136static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1137{
93a23a72
YZ		 1138 if (!info)
1139 return;
1140
1141 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1142}
1143
1144static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1145{
1146 if (!info->dev || !pci_ats_enabled(info->dev))
1147 return;
1148
1149 pci_disable_ats(info->dev);
1150}
1151
1152static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1153 u64 addr, unsigned mask)
1154{
1155 u16 sid, qdep;
1156 unsigned long flags;
1157 struct device_domain_info *info;
1158
1159 spin_lock_irqsave(&device_domain_lock, flags);
1160 list_for_each_entry(info, &domain->devices, link) {
1161 if (!info->dev || !pci_ats_enabled(info->dev))
1162 continue;
1163
1164 sid = info->bus << 8 | info->devfn;
1165 qdep = pci_ats_queue_depth(info->dev);
1166 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1167 }
1168 spin_unlock_irqrestore(&device_domain_lock, flags);
1169}
1170
1f0ef2aa 1171static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1172 unsigned long pfn, unsigned int pages, int map)
ba395927 1173{
9dd2fe89 1174 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1175 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1176
ba395927
KA		 1177 BUG_ON(pages == 0);
1178
ba395927 1179 /*
9dd2fe89
YZ		 1180 * Fallback to domain selective flush if no PSI support or the size is
1181 * too big.
ba395927
KA		 1182 * PSI requires page size to be 2 ^ x, and the base address is naturally
1183 * aligned to the size
1184 */
9dd2fe89
YZ		 1185 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1186 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1187 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1188 else
1189 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1190 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1191
1192 /*
82653633
NA		 1193 * In caching mode, changes of pages from non-present to present require
1194 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1195 */
82653633 1196 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1197 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1198}
1199
f8bab735 1200static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1201{
1202 u32 pmen;
1203 unsigned long flags;
1204
1f5b3c3f 1205 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1206 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1207 pmen &= ~DMA_PMEN_EPM;
1208 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1209
1210 /* wait for the protected region status bit to clear */
1211 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1212 readl, !(pmen & DMA_PMEN_PRS), pmen);
1213
1f5b3c3f 1214 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1215}
1216
ba395927
KA		 1217static int iommu_enable_translation(struct intel_iommu *iommu)
1218{
1219 u32 sts;
1220 unsigned long flags;
1221
1f5b3c3f 1222 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1223 iommu->gcmd |= DMA_GCMD_TE;
1224 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1225
1226 /* Make sure hardware complete it */
1227 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1228 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1229
1f5b3c3f 1230 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1231 return 0;
1232}
1233
1234static int iommu_disable_translation(struct intel_iommu *iommu)
1235{
1236 u32 sts;
1237 unsigned long flag;
1238
1f5b3c3f 1239 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1240 iommu->gcmd &= ~DMA_GCMD_TE;
1241 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1242
1243 /* Make sure hardware complete it */
1244 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1245 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1246
1f5b3c3f 1247 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1248 return 0;
1249}
1250
3460a6d9 1251
ba395927
KA		 1252static int iommu_init_domains(struct intel_iommu *iommu)
1253{
1254 unsigned long ndomains;
1255 unsigned long nlongs;
1256
1257 ndomains = cap_ndoms(iommu->cap);
68aeb968 1258 pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
680a7524 1259 ndomains);
ba395927
KA		 1260 nlongs = BITS_TO_LONGS(ndomains);
1261
94a91b50
DD		 1262 spin_lock_init(&iommu->lock);
1263
ba395927
KA		 1264 /* TBD: there might be 64K domains,
1265 * consider other allocation for future chip
1266 */
1267 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1268 if (!iommu->domain_ids) {
1269 printk(KERN_ERR "Allocating domain id array failed\n");
1270 return -ENOMEM;
1271 }
1272 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1273 GFP_KERNEL);
1274 if (!iommu->domains) {
1275 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1276 return -ENOMEM;
1277 }
1278
1279 /*
1280 * if Caching mode is set, then invalid translations are tagged
1281 * with domainid 0. Hence we need to pre-allocate it.
1282 */
1283 if (cap_caching_mode(iommu->cap))
1284 set_bit(0, iommu->domain_ids);
1285 return 0;
1286}
ba395927 1287
ba395927
KA		 1288
1289static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1290static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1291
1292void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1293{
1294 struct dmar_domain *domain;
1295 int i;
c7151a8d 1296 unsigned long flags;
ba395927 1297
94a91b50 1298 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1299 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1300 domain = iommu->domains[i];
1301 clear_bit(i, iommu->domain_ids);
1302
1303 spin_lock_irqsave(&domain->iommu_lock, flags);
1304 if (--domain->iommu_count == 0) {
1305 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1306 vm_domain_exit(domain);
1307 else
1308 domain_exit(domain);
1309 }
1310 spin_unlock_irqrestore(&domain->iommu_lock, flags);
5e98c4b1 1311 }
ba395927
KA		 1312 }
1313
1314 if (iommu->gcmd & DMA_GCMD_TE)
1315 iommu_disable_translation(iommu);
1316
1317 if (iommu->irq) {
dced35ae 1318 irq_set_handler_data(iommu->irq, NULL);
ba395927
KA		 1319 /* This will mask the irq */
1320 free_irq(iommu->irq, iommu);
1321 destroy_irq(iommu->irq);
1322 }
1323
1324 kfree(iommu->domains);
1325 kfree(iommu->domain_ids);
1326
d9630fe9
WH		 1327 g_iommus[iommu->seq_id] = NULL;
1328
1329 /* if all iommus are freed, free g_iommus */
1330 for (i = 0; i < g_num_of_iommus; i++) {
1331 if (g_iommus[i])
1332 break;
1333 }
1334
1335 if (i == g_num_of_iommus)
1336 kfree(g_iommus);
1337
ba395927
KA		 1338 /* free context mapping */
1339 free_context_table(iommu);
ba395927
KA		 1340}
1341
2c2e2c38 1342static struct dmar_domain *alloc_domain(void)
ba395927 1343{
ba395927 1344 struct dmar_domain *domain;
ba395927
KA		 1345
1346 domain = alloc_domain_mem();
1347 if (!domain)
1348 return NULL;
1349
4c923d47 1350 domain->nid = -1;
1b198bb0 1351 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38
FY		 1352 domain->flags = 0;
1353
1354 return domain;
1355}
1356
1357static int iommu_attach_domain(struct dmar_domain *domain,
1358 struct intel_iommu *iommu)
1359{
1360 int num;
1361 unsigned long ndomains;
1362 unsigned long flags;
1363
ba395927
KA		 1364 ndomains = cap_ndoms(iommu->cap);
1365
1366 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1367
ba395927
KA		 1368 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1369 if (num >= ndomains) {
1370 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1371 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1372 return -ENOMEM;
ba395927
KA		 1373 }
1374
ba395927 1375 domain->id = num;
2c2e2c38 1376 set_bit(num, iommu->domain_ids);
1b198bb0 1377 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1378 iommu->domains[num] = domain;
1379 spin_unlock_irqrestore(&iommu->lock, flags);
1380
2c2e2c38 1381 return 0;
ba395927
KA		 1382}
1383
2c2e2c38
FY		 1384static void iommu_detach_domain(struct dmar_domain *domain,
1385 struct intel_iommu *iommu)
ba395927
KA		 1386{
1387 unsigned long flags;
2c2e2c38
FY		 1388 int num, ndomains;
1389 int found = 0;
ba395927 1390
8c11e798 1391 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1392 ndomains = cap_ndoms(iommu->cap);
a45946ab 1393 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1394 if (iommu->domains[num] == domain) {
1395 found = 1;
1396 break;
1397 }
2c2e2c38
FY		 1398 }
1399
1400 if (found) {
1401 clear_bit(num, iommu->domain_ids);
1b198bb0 1402 clear_bit(iommu->seq_id, domain->iommu_bmp);
2c2e2c38
FY		 1403 iommu->domains[num] = NULL;
1404 }
8c11e798 1405 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1406}
1407
1408static struct iova_domain reserved_iova_list;
8a443df4 1409static struct lock_class_key reserved_rbtree_key;
ba395927 1410
51a63e67 1411static int dmar_init_reserved_ranges(void)
ba395927
KA		 1412{
1413 struct pci_dev *pdev = NULL;
1414 struct iova *iova;
1415 int i;
ba395927 1416
f661197e 1417 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1418
8a443df4
MG		 1419 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1420 &reserved_rbtree_key);
1421
ba395927
KA		 1422 /* IOAPIC ranges shouldn't be accessed by DMA */
1423 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1424 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1425 if (!iova) {
ba395927 1426 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1427 return -ENODEV;
1428 }
ba395927
KA		 1429
1430 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1431 for_each_pci_dev(pdev) {
1432 struct resource *r;
1433
1434 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1435 r = &pdev->resource[i];
1436 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1437 continue;
1a4a4551
DW		 1438 iova = reserve_iova(&reserved_iova_list,
1439 IOVA_PFN(r->start),
1440 IOVA_PFN(r->end));
51a63e67 1441 if (!iova) {
ba395927 1442 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1443 return -ENODEV;
1444 }
ba395927
KA		 1445 }
1446 }
51a63e67 1447 return 0;
ba395927
KA		 1448}
1449
1450static void domain_reserve_special_ranges(struct dmar_domain *domain)
1451{
1452 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1453}
1454
1455static inline int guestwidth_to_adjustwidth(int gaw)
1456{
1457 int agaw;
1458 int r = (gaw - 12) % 9;
1459
1460 if (r == 0)
1461 agaw = gaw;
1462 else
1463 agaw = gaw + 9 - r;
1464 if (agaw > 64)
1465 agaw = 64;
1466 return agaw;
1467}
1468
1469static int domain_init(struct dmar_domain *domain, int guest_width)
1470{
1471 struct intel_iommu *iommu;
1472 int adjust_width, agaw;
1473 unsigned long sagaw;
1474
f661197e 1475 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1476 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1477
1478 domain_reserve_special_ranges(domain);
1479
1480 /* calculate AGAW */
8c11e798 1481 iommu = domain_get_iommu(domain);
ba395927
KA		 1482 if (guest_width > cap_mgaw(iommu->cap))
1483 guest_width = cap_mgaw(iommu->cap);
1484 domain->gaw = guest_width;
1485 adjust_width = guestwidth_to_adjustwidth(guest_width);
1486 agaw = width_to_agaw(adjust_width);
1487 sagaw = cap_sagaw(iommu->cap);
1488 if (!test_bit(agaw, &sagaw)) {
1489 /* hardware doesn't support it, choose a bigger one */
1490 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1491 agaw = find_next_bit(&sagaw, 5, agaw);
1492 if (agaw >= 5)
1493 return -ENODEV;
1494 }
1495 domain->agaw = agaw;
1496 INIT_LIST_HEAD(&domain->devices);
1497
8e604097
WH		 1498 if (ecap_coherent(iommu->ecap))
1499 domain->iommu_coherency = 1;
1500 else
1501 domain->iommu_coherency = 0;
1502
58c610bd
SY		 1503 if (ecap_sc_support(iommu->ecap))
1504 domain->iommu_snooping = 1;
1505 else
1506 domain->iommu_snooping = 0;
1507
6dd9a7c7 1508 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
c7151a8d 1509 domain->iommu_count = 1;
4c923d47 1510 domain->nid = iommu->node;
c7151a8d 1511
ba395927 1512 /* always allocate the top pgd */
4c923d47 1513 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1514 if (!domain->pgd)
1515 return -ENOMEM;
5b6985ce 1516 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1517 return 0;
1518}
1519
1520static void domain_exit(struct dmar_domain *domain)
1521{
2c2e2c38
FY		 1522 struct dmar_drhd_unit *drhd;
1523 struct intel_iommu *iommu;
ba395927
KA		 1524
1525 /* Domain 0 is reserved, so dont process it */
1526 if (!domain)
1527 return;
1528
7b668357
AW		 1529 /* Flush any lazy unmaps that may reference this domain */
1530 if (!intel_iommu_strict)
1531 flush_unmaps_timeout(0);
1532
ba395927
KA		 1533 domain_remove_dev_info(domain);
1534 /* destroy iovas */
1535 put_iova_domain(&domain->iovad);
ba395927
KA		 1536
1537 /* clear ptes */
595badf5 1538 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1539
1540 /* free page tables */
d794dc9b 1541 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1542
2c2e2c38 1543 for_each_active_iommu(iommu, drhd)
1b198bb0 1544 if (test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38
FY		 1545 iommu_detach_domain(domain, iommu);
1546
ba395927
KA		 1547 free_domain_mem(domain);
1548}
1549
4ed0d3e6
FY		 1550static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1551 u8 bus, u8 devfn, int translation)
ba395927
KA		 1552{
1553 struct context_entry *context;
ba395927 1554 unsigned long flags;
5331fe6f 1555 struct intel_iommu *iommu;
ea6606b0
WH		 1556 struct dma_pte *pgd;
1557 unsigned long num;
1558 unsigned long ndomains;
1559 int id;
1560 int agaw;
93a23a72 1561 struct device_domain_info *info = NULL;
ba395927
KA		 1562
1563 pr_debug("Set context mapping for %02x:%02x.%d\n",
1564 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1565
ba395927 1566 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1567 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1568 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1569
276dbf99 1570 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1571 if (!iommu)
1572 return -ENODEV;
1573
ba395927
KA		 1574 context = device_to_context_entry(iommu, bus, devfn);
1575 if (!context)
1576 return -ENOMEM;
1577 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1578 if (context_present(context)) {
ba395927
KA		 1579 spin_unlock_irqrestore(&iommu->lock, flags);
1580 return 0;
1581 }
1582
ea6606b0
WH		 1583 id = domain->id;
1584 pgd = domain->pgd;
1585
2c2e2c38
FY		 1586 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1587 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1588 int found = 0;
1589
1590 /* find an available domain id for this device in iommu */
1591 ndomains = cap_ndoms(iommu->cap);
a45946ab 1592 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1593 if (iommu->domains[num] == domain) {
1594 id = num;
1595 found = 1;
1596 break;
1597 }
ea6606b0
WH		 1598 }
1599
1600 if (found == 0) {
1601 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1602 if (num >= ndomains) {
1603 spin_unlock_irqrestore(&iommu->lock, flags);
1604 printk(KERN_ERR "IOMMU: no free domain ids\n");
1605 return -EFAULT;
1606 }
1607
1608 set_bit(num, iommu->domain_ids);
1609 iommu->domains[num] = domain;
1610 id = num;
1611 }
1612
1613 /* Skip top levels of page tables for
1614 * iommu which has less agaw than default.
1672af11 1615 * Unnecessary for PT mode.
ea6606b0 1616 */
1672af11
CW		 1617 if (translation != CONTEXT_TT_PASS_THROUGH) {
1618 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1619 pgd = phys_to_virt(dma_pte_addr(pgd));
1620 if (!dma_pte_present(pgd)) {
1621 spin_unlock_irqrestore(&iommu->lock, flags);
1622 return -ENOMEM;
1623 }
ea6606b0
WH		 1624 }
1625 }
1626 }
1627
1628 context_set_domain_id(context, id);
4ed0d3e6 1629
93a23a72
YZ		 1630 if (translation != CONTEXT_TT_PASS_THROUGH) {
1631 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1632 translation = info ? CONTEXT_TT_DEV_IOTLB :
1633 CONTEXT_TT_MULTI_LEVEL;
1634 }
4ed0d3e6
FY		 1635 /*
1636 * In pass through mode, AW must be programmed to indicate the largest
1637 * AGAW value supported by hardware. And ASR is ignored by hardware.
1638 */
93a23a72 1639 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1640 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1641 else {
1642 context_set_address_root(context, virt_to_phys(pgd));
1643 context_set_address_width(context, iommu->agaw);
1644 }
4ed0d3e6
FY		 1645
1646 context_set_translation_type(context, translation);
c07e7d21
MM		 1647 context_set_fault_enable(context);
1648 context_set_present(context);
5331fe6f 1649 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1650
4c25a2c1
DW		 1651 /*
1652 * It's a non-present to present mapping. If hardware doesn't cache
1653 * non-present entry we only need to flush the write-buffer. If the
1654 * _does_ cache non-present entries, then it does so in the special
1655 * domain #0, which we have to flush:
1656 */
1657 if (cap_caching_mode(iommu->cap)) {
1658 iommu->flush.flush_context(iommu, 0,
1659 (((u16)bus) << 8) | devfn,
1660 DMA_CCMD_MASK_NOBIT,
1661 DMA_CCMD_DEVICE_INVL);
82653633 1662 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1663 } else {
ba395927 1664 iommu_flush_write_buffer(iommu);
4c25a2c1 1665 }
93a23a72 1666 iommu_enable_dev_iotlb(info);
ba395927 1667 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1668
1669 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1670 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1671 domain->iommu_count++;
4c923d47
SS		 1672 if (domain->iommu_count == 1)
1673 domain->nid = iommu->node;
58c610bd 1674 domain_update_iommu_cap(domain);
c7151a8d
WH		 1675 }
1676 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1677 return 0;
1678}
1679
1680static int
4ed0d3e6
FY		 1681domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1682 int translation)
ba395927
KA		 1683{
1684 int ret;
1685 struct pci_dev *tmp, *parent;
1686
276dbf99 1687 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1688 pdev->bus->number, pdev->devfn,
1689 translation);
ba395927
KA		 1690 if (ret)
1691 return ret;
1692
1693 /* dependent device mapping */
1694 tmp = pci_find_upstream_pcie_bridge(pdev);
1695 if (!tmp)
1696 return 0;
1697 /* Secondary interface's bus number and devfn 0 */
1698 parent = pdev->bus->self;
1699 while (parent != tmp) {
276dbf99
DW		 1700 ret = domain_context_mapping_one(domain,
1701 pci_domain_nr(parent->bus),
1702 parent->bus->number,
4ed0d3e6 1703 parent->devfn, translation);
ba395927
KA		 1704 if (ret)
1705 return ret;
1706 parent = parent->bus->self;
1707 }
45e829ea 1708 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1709 return domain_context_mapping_one(domain,
276dbf99 1710 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1711 tmp->subordinate->number, 0,
1712 translation);
ba395927
KA		 1713 else /* this is a legacy PCI bridge */
1714 return domain_context_mapping_one(domain,
276dbf99
DW		 1715 pci_domain_nr(tmp->bus),
1716 tmp->bus->number,
4ed0d3e6
FY		 1717 tmp->devfn,
1718 translation);
ba395927
KA		 1719}
1720
5331fe6f 1721static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1722{
1723 int ret;
1724 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1725 struct intel_iommu *iommu;
1726
276dbf99
DW		 1727 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1728 pdev->devfn);
5331fe6f
WH		 1729 if (!iommu)
1730 return -ENODEV;
ba395927 1731
276dbf99 1732 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1733 if (!ret)
1734 return ret;
1735 /* dependent device mapping */
1736 tmp = pci_find_upstream_pcie_bridge(pdev);
1737 if (!tmp)
1738 return ret;
1739 /* Secondary interface's bus number and devfn 0 */
1740 parent = pdev->bus->self;
1741 while (parent != tmp) {
8c11e798 1742 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1743 parent->devfn);
ba395927
KA		 1744 if (!ret)
1745 return ret;
1746 parent = parent->bus->self;
1747 }
5f4d91a1 1748 if (pci_is_pcie(tmp))
276dbf99
DW		 1749 return device_context_mapped(iommu, tmp->subordinate->number,
1750 0);
ba395927 1751 else
276dbf99
DW		 1752 return device_context_mapped(iommu, tmp->bus->number,
1753 tmp->devfn);
ba395927
KA		 1754}
1755
f532959b
FY		 1756/* Returns a number of VTD pages, but aligned to MM page size */
1757static inline unsigned long aligned_nrpages(unsigned long host_addr,
1758 size_t size)
1759{
1760 host_addr &= ~PAGE_MASK;
1761 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1762}
1763
6dd9a7c7
YS		 1764/* Return largest possible superpage level for a given mapping */
1765static inline int hardware_largepage_caps(struct dmar_domain *domain,
1766 unsigned long iov_pfn,
1767 unsigned long phy_pfn,
1768 unsigned long pages)
1769{
1770 int support, level = 1;
1771 unsigned long pfnmerge;
1772
1773 support = domain->iommu_superpage;
1774
1775 /* To use a large page, the virtual *and* physical addresses
1776 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1777 of them will mean we have to use smaller pages. So just
1778 merge them and check both at once. */
1779 pfnmerge = iov_pfn | phy_pfn;
1780
1781 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1782 pages >>= VTD_STRIDE_SHIFT;
1783 if (!pages)
1784 break;
1785 pfnmerge >>= VTD_STRIDE_SHIFT;
1786 level++;
1787 support--;
1788 }
1789 return level;
1790}
1791
9051aa02
DW		 1792static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1793 struct scatterlist *sg, unsigned long phys_pfn,
1794 unsigned long nr_pages, int prot)
e1605495
DW		 1795{
1796 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1797 phys_addr_t uninitialized_var(pteval);
e1605495 1798 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
b658f2ad 1799 unsigned long sg_res = 0;
6dd9a7c7
YS		 1800 unsigned int largepage_lvl = 0;
1801 unsigned long lvl_pages = 0;
e1605495
DW		 1802
1803 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1804
1805 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1806 return -EINVAL;
1807
1808 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1809
b658f2ad
JL		 1810 if (!sg) {
1811 sg_res = nr_pages;
9051aa02
DW		 1812 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1813 }
1814
6dd9a7c7 1815 while (nr_pages > 0) {
c85994e4
DW		 1816 uint64_t tmp;
1817
e1605495 1818 if (!sg_res) {
f532959b 1819 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1820 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1821 sg->dma_length = sg->length;
1822 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1823 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1824 }
6dd9a7c7 1825
e1605495 1826 if (!pte) {
6dd9a7c7
YS		 1827 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1828
1829 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
e1605495
DW		 1830 if (!pte)
1831 return -ENOMEM;
6dd9a7c7 1832 /* It is large page*/
6491d4d0 1833 if (largepage_lvl > 1) {
6dd9a7c7 1834 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1835 /* Ensure that old small page tables are removed to make room
1836 for superpage, if they exist. */
1837 dma_pte_clear_range(domain, iov_pfn,
1838 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1839 dma_pte_free_pagetable(domain, iov_pfn,
1840 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1841 } else {
6dd9a7c7 1842 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1843 }
6dd9a7c7 1844
e1605495
DW		 1845 }
1846 /* We don't need lock here, nobody else
1847 * touches the iova range
1848 */
7766a3fb 1849 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1850 if (tmp) {
1bf20f0d 1851 static int dumps = 5;
c85994e4
DW		 1852 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1853 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1854 if (dumps) {
1855 dumps--;
1856 debug_dma_dump_mappings(NULL);
1857 }
1858 WARN_ON(1);
1859 }
6dd9a7c7
YS		 1860
1861 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1862
1863 BUG_ON(nr_pages < lvl_pages);
1864 BUG_ON(sg_res < lvl_pages);
1865
1866 nr_pages -= lvl_pages;
1867 iov_pfn += lvl_pages;
1868 phys_pfn += lvl_pages;
1869 pteval += lvl_pages * VTD_PAGE_SIZE;
1870 sg_res -= lvl_pages;
1871
1872 /* If the next PTE would be the first in a new page, then we
1873 need to flush the cache on the entries we've just written.
1874 And then we'll need to recalculate 'pte', so clear it and
1875 let it get set again in the if (!pte) block above.
1876
1877 If we're done (!nr_pages) we need to flush the cache too.
1878
1879 Also if we've been setting superpages, we may need to
1880 recalculate 'pte' and switch back to smaller pages for the
1881 end of the mapping, if the trailing size is not enough to
1882 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 1883 pte++;
6dd9a7c7
YS		 1884 if (!nr_pages || first_pte_in_page(pte) ||
1885 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 1886 domain_flush_cache(domain, first_pte,
1887 (void *)pte - (void *)first_pte);
1888 pte = NULL;
1889 }
6dd9a7c7
YS		 1890
1891 if (!sg_res && nr_pages)
e1605495
DW		 1892 sg = sg_next(sg);
1893 }
1894 return 0;
1895}
1896
9051aa02
DW		 1897static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1898 struct scatterlist *sg, unsigned long nr_pages,
1899 int prot)
ba395927 1900{
9051aa02
DW		 1901 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1902}
6f6a00e4 1903
9051aa02
DW		 1904static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1905 unsigned long phys_pfn, unsigned long nr_pages,
1906 int prot)
1907{
1908 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1909}
1910
c7151a8d 1911static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1912{
c7151a8d
WH		 1913 if (!iommu)
1914 return;
8c11e798
WH		 1915
1916 clear_context_table(iommu, bus, devfn);
1917 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1918 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1919 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1920}
1921
109b9b04
DW		 1922static inline void unlink_domain_info(struct device_domain_info *info)
1923{
1924 assert_spin_locked(&device_domain_lock);
1925 list_del(&info->link);
1926 list_del(&info->global);
1927 if (info->dev)
1928 info->dev->dev.archdata.iommu = NULL;
1929}
1930
ba395927
KA		 1931static void domain_remove_dev_info(struct dmar_domain *domain)
1932{
1933 struct device_domain_info *info;
1934 unsigned long flags;
c7151a8d 1935 struct intel_iommu *iommu;
ba395927
KA		 1936
1937 spin_lock_irqsave(&device_domain_lock, flags);
1938 while (!list_empty(&domain->devices)) {
1939 info = list_entry(domain->devices.next,
1940 struct device_domain_info, link);
109b9b04 1941 unlink_domain_info(info);
ba395927
KA		 1942 spin_unlock_irqrestore(&device_domain_lock, flags);
1943
93a23a72 1944 iommu_disable_dev_iotlb(info);
276dbf99 1945 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1946 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1947 free_devinfo_mem(info);
1948
1949 spin_lock_irqsave(&device_domain_lock, flags);
1950 }
1951 spin_unlock_irqrestore(&device_domain_lock, flags);
1952}
1953
1954/*
1955 * find_domain
358dd8ac 1956 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1957 */
38717946 1958static struct dmar_domain *
ba395927
KA		 1959find_domain(struct pci_dev *pdev)
1960{
1961 struct device_domain_info *info;
1962
1963 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1964 info = pdev->dev.archdata.iommu;
ba395927
KA		 1965 if (info)
1966 return info->domain;
1967 return NULL;
1968}
1969
ba395927
KA		 1970/* domain is initialized */
1971static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1972{
1973 struct dmar_domain *domain, *found = NULL;
1974 struct intel_iommu *iommu;
1975 struct dmar_drhd_unit *drhd;
1976 struct device_domain_info *info, *tmp;
1977 struct pci_dev *dev_tmp;
1978 unsigned long flags;
1979 int bus = 0, devfn = 0;
276dbf99 1980 int segment;
2c2e2c38 1981 int ret;
ba395927
KA		 1982
1983 domain = find_domain(pdev);
1984 if (domain)
1985 return domain;
1986
276dbf99
DW		 1987 segment = pci_domain_nr(pdev->bus);
1988
ba395927
KA		 1989 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1990 if (dev_tmp) {
5f4d91a1 1991 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1992 bus = dev_tmp->subordinate->number;
1993 devfn = 0;
1994 } else {
1995 bus = dev_tmp->bus->number;
1996 devfn = dev_tmp->devfn;
1997 }
1998 spin_lock_irqsave(&device_domain_lock, flags);
1999 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 2000 if (info->segment == segment &&
2001 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 2002 found = info->domain;
2003 break;
2004 }
2005 }
2006 spin_unlock_irqrestore(&device_domain_lock, flags);
2007 /* pcie-pci bridge already has a domain, uses it */
2008 if (found) {
2009 domain = found;
2010 goto found_domain;
2011 }
2012 }
2013
2c2e2c38
FY		 2014 domain = alloc_domain();
2015 if (!domain)
2016 goto error;
2017
ba395927
KA		 2018 /* Allocate new domain for the device */
2019 drhd = dmar_find_matched_drhd_unit(pdev);
2020 if (!drhd) {
2021 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2022 pci_name(pdev));
d2900bd6 2023 free_domain_mem(domain);
ba395927
KA		 2024 return NULL;
2025 }
2026 iommu = drhd->iommu;
2027
2c2e2c38
FY		 2028 ret = iommu_attach_domain(domain, iommu);
2029 if (ret) {
2fe9723d 2030 free_domain_mem(domain);
ba395927 2031 goto error;
2c2e2c38 2032 }
ba395927
KA		 2033
2034 if (domain_init(domain, gaw)) {
2035 domain_exit(domain);
2036 goto error;
2037 }
2038
2039 /* register pcie-to-pci device */
2040 if (dev_tmp) {
2041 info = alloc_devinfo_mem();
2042 if (!info) {
2043 domain_exit(domain);
2044 goto error;
2045 }
276dbf99 2046 info->segment = segment;
ba395927
KA		 2047 info->bus = bus;
2048 info->devfn = devfn;
2049 info->dev = NULL;
2050 info->domain = domain;
2051 /* This domain is shared by devices under p2p bridge */
3b5410e7 2052 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 2053
2054 /* pcie-to-pci bridge already has a domain, uses it */
2055 found = NULL;
2056 spin_lock_irqsave(&device_domain_lock, flags);
2057 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 2058 if (tmp->segment == segment &&
2059 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 2060 found = tmp->domain;
2061 break;
2062 }
2063 }
2064 if (found) {
00dfff77 2065 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2066 free_devinfo_mem(info);
2067 domain_exit(domain);
2068 domain = found;
2069 } else {
2070 list_add(&info->link, &domain->devices);
2071 list_add(&info->global, &device_domain_list);
00dfff77 2072 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2073 }
ba395927
KA		 2074 }
2075
2076found_domain:
2077 info = alloc_devinfo_mem();
2078 if (!info)
2079 goto error;
276dbf99 2080 info->segment = segment;
ba395927
KA		 2081 info->bus = pdev->bus->number;
2082 info->devfn = pdev->devfn;
2083 info->dev = pdev;
2084 info->domain = domain;
2085 spin_lock_irqsave(&device_domain_lock, flags);
2086 /* somebody is fast */
2087 found = find_domain(pdev);
2088 if (found != NULL) {
2089 spin_unlock_irqrestore(&device_domain_lock, flags);
2090 if (found != domain) {
2091 domain_exit(domain);
2092 domain = found;
2093 }
2094 free_devinfo_mem(info);
2095 return domain;
2096 }
2097 list_add(&info->link, &domain->devices);
2098 list_add(&info->global, &device_domain_list);
358dd8ac 2099 pdev->dev.archdata.iommu = info;
ba395927
KA		 2100 spin_unlock_irqrestore(&device_domain_lock, flags);
2101 return domain;
2102error:
2103 /* recheck it here, maybe others set it */
2104 return find_domain(pdev);
2105}
2106
2c2e2c38 2107static int iommu_identity_mapping;
e0fc7e0b
DW		 2108#define IDENTMAP_ALL 1
2109#define IDENTMAP_GFX 2
2110#define IDENTMAP_AZALIA 4
2c2e2c38 2111
b213203e
DW		 2112static int iommu_domain_identity_map(struct dmar_domain *domain,
2113 unsigned long long start,
2114 unsigned long long end)
ba395927 2115{
c5395d5c
DW		 2116 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2117 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2118
2119 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2120 dma_to_mm_pfn(last_vpfn))) {
ba395927 2121 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2122 return -ENOMEM;
ba395927
KA		 2123 }
2124
c5395d5c
DW		 2125 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2126 start, end, domain->id);
ba395927
KA		 2127 /*
2128 * RMRR range might have overlap with physical memory range,
2129 * clear it first
2130 */
c5395d5c 2131 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2132
c5395d5c
DW		 2133 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2134 last_vpfn - first_vpfn + 1,
61df7443 2135 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2136}
2137
2138static int iommu_prepare_identity_map(struct pci_dev *pdev,
2139 unsigned long long start,
2140 unsigned long long end)
2141{
2142 struct dmar_domain *domain;
2143 int ret;
2144
c7ab48d2 2145 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2146 if (!domain)
2147 return -ENOMEM;
2148
19943b0e
DW		 2149 /* For _hardware_ passthrough, don't bother. But for software
2150 passthrough, we do it anyway -- it may indicate a memory
2151 range which is reserved in E820, so which didn't get set
2152 up to start with in si_domain */
2153 if (domain == si_domain && hw_pass_through) {
2154 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2155 pci_name(pdev), start, end);
2156 return 0;
2157 }
2158
2159 printk(KERN_INFO
2160 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2161 pci_name(pdev), start, end);
2ff729f5 2162
5595b528
DW		 2163 if (end < start) {
2164 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2165 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2166 dmi_get_system_info(DMI_BIOS_VENDOR),
2167 dmi_get_system_info(DMI_BIOS_VERSION),
2168 dmi_get_system_info(DMI_PRODUCT_VERSION));
2169 ret = -EIO;
2170 goto error;
2171 }
2172
2ff729f5
DW		 2173 if (end >> agaw_to_width(domain->agaw)) {
2174 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2175 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2176 agaw_to_width(domain->agaw),
2177 dmi_get_system_info(DMI_BIOS_VENDOR),
2178 dmi_get_system_info(DMI_BIOS_VERSION),
2179 dmi_get_system_info(DMI_PRODUCT_VERSION));
2180 ret = -EIO;
2181 goto error;
2182 }
19943b0e 2183
b213203e 2184 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2185 if (ret)
2186 goto error;
2187
2188 /* context entry init */
4ed0d3e6 2189 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2190 if (ret)
2191 goto error;
2192
2193 return 0;
2194
2195 error:
ba395927
KA		 2196 domain_exit(domain);
2197 return ret;
ba395927
KA		 2198}
2199
2200static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2201 struct pci_dev *pdev)
2202{
358dd8ac 2203 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2204 return 0;
2205 return iommu_prepare_identity_map(pdev, rmrr->base_address,
70e535d1 2206 rmrr->end_address);
ba395927
KA		 2207}
2208
d3f13810 2209#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2210static inline void iommu_prepare_isa(void)
2211{
2212 struct pci_dev *pdev;
2213 int ret;
2214
2215 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2216 if (!pdev)
2217 return;
2218
c7ab48d2 2219 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
70e535d1 2220 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2221
2222 if (ret)
c7ab48d2
DW		 2223 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2224 "floppy might not work\n");
49a0429e
KA		 2225
2226}
2227#else
2228static inline void iommu_prepare_isa(void)
2229{
2230 return;
2231}
d3f13810 2232#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2233
2c2e2c38 2234static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2235
071e1374 2236static int __init si_domain_init(int hw)
2c2e2c38
FY		 2237{
2238 struct dmar_drhd_unit *drhd;
2239 struct intel_iommu *iommu;
c7ab48d2 2240 int nid, ret = 0;
2c2e2c38
FY		 2241
2242 si_domain = alloc_domain();
2243 if (!si_domain)
2244 return -EFAULT;
2245
c7ab48d2 2246 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2247
2248 for_each_active_iommu(iommu, drhd) {
2249 ret = iommu_attach_domain(si_domain, iommu);
2250 if (ret) {
2251 domain_exit(si_domain);
2252 return -EFAULT;
2253 }
2254 }
2255
2256 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2257 domain_exit(si_domain);
2258 return -EFAULT;
2259 }
2260
2261 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2262
19943b0e
DW		 2263 if (hw)
2264 return 0;
2265
c7ab48d2 2266 for_each_online_node(nid) {
5dfe8660
TH		 2267 unsigned long start_pfn, end_pfn;
2268 int i;
2269
2270 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2271 ret = iommu_domain_identity_map(si_domain,
2272 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2273 if (ret)
2274 return ret;
2275 }
c7ab48d2
DW		 2276 }
2277
2c2e2c38
FY		 2278 return 0;
2279}
2280
2281static void domain_remove_one_dev_info(struct dmar_domain *domain,
2282 struct pci_dev *pdev);
2283static int identity_mapping(struct pci_dev *pdev)
2284{
2285 struct device_domain_info *info;
2286
2287 if (likely(!iommu_identity_mapping))
2288 return 0;
2289
cb452a40
MT		 2290 info = pdev->dev.archdata.iommu;
2291 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2292 return (info->domain == si_domain);
2c2e2c38 2293
2c2e2c38
FY		 2294 return 0;
2295}
2296
2297static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2298 struct pci_dev *pdev,
2299 int translation)
2c2e2c38
FY		 2300{
2301 struct device_domain_info *info;
2302 unsigned long flags;
5fe60f4e 2303 int ret;
2c2e2c38
FY		 2304
2305 info = alloc_devinfo_mem();
2306 if (!info)
2307 return -ENOMEM;
2308
2309 info->segment = pci_domain_nr(pdev->bus);
2310 info->bus = pdev->bus->number;
2311 info->devfn = pdev->devfn;
2312 info->dev = pdev;
2313 info->domain = domain;
2314
2315 spin_lock_irqsave(&device_domain_lock, flags);
2316 list_add(&info->link, &domain->devices);
2317 list_add(&info->global, &device_domain_list);
2318 pdev->dev.archdata.iommu = info;
2319 spin_unlock_irqrestore(&device_domain_lock, flags);
2320
e2ad23d0
DW		 2321 ret = domain_context_mapping(domain, pdev, translation);
2322 if (ret) {
2323 spin_lock_irqsave(&device_domain_lock, flags);
109b9b04 2324 unlink_domain_info(info);
e2ad23d0
DW		 2325 spin_unlock_irqrestore(&device_domain_lock, flags);
2326 free_devinfo_mem(info);
2327 return ret;
2328 }
2329
2c2e2c38
FY		 2330 return 0;
2331}
2332
ea2447f7
TM		 2333static bool device_has_rmrr(struct pci_dev *dev)
2334{
2335 struct dmar_rmrr_unit *rmrr;
2336 int i;
2337
2338 for_each_rmrr_units(rmrr) {
2339 for (i = 0; i < rmrr->devices_cnt; i++) {
2340 /*
2341 * Return TRUE if this RMRR contains the device that
2342 * is passed in.
2343 */
2344 if (rmrr->devices[i] == dev)
2345 return true;
2346 }
2347 }
2348 return false;
2349}
2350
6941af28
DW		 2351static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2352{
ea2447f7
TM		 2353
2354 /*
2355 * We want to prevent any device associated with an RMRR from
2356 * getting placed into the SI Domain. This is done because
2357 * problems exist when devices are moved in and out of domains
2358 * and their respective RMRR info is lost. We exempt USB devices
2359 * from this process due to their usage of RMRRs that are known
2360 * to not be needed after BIOS hand-off to OS.
2361 */
2362 if (device_has_rmrr(pdev) &&
2363 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2364 return 0;
2365
e0fc7e0b
DW		 2366 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2367 return 1;
2368
2369 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2370 return 1;
2371
2372 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2373 return 0;
6941af28 2374
3dfc813d
DW		 2375 /*
2376 * We want to start off with all devices in the 1:1 domain, and
2377 * take them out later if we find they can't access all of memory.
2378 *
2379 * However, we can't do this for PCI devices behind bridges,
2380 * because all PCI devices behind the same bridge will end up
2381 * with the same source-id on their transactions.
2382 *
2383 * Practically speaking, we can't change things around for these
2384 * devices at run-time, because we can't be sure there'll be no
2385 * DMA transactions in flight for any of their siblings.
2386 *
2387 * So PCI devices (unless they're on the root bus) as well as
2388 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2389 * the 1:1 domain, just in _case_ one of their siblings turns out
2390 * not to be able to map all of memory.
2391 */
5f4d91a1 2392 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2393 if (!pci_is_root_bus(pdev->bus))
2394 return 0;
2395 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2396 return 0;
62f87c0e 2397 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d
DW		 2398 return 0;
2399
2400 /*
2401 * At boot time, we don't yet know if devices will be 64-bit capable.
2402 * Assume that they will -- if they turn out not to be, then we can
2403 * take them out of the 1:1 domain later.
2404 */
8fcc5372
CW		 2405 if (!startup) {
2406 /*
2407 * If the device's dma_mask is less than the system's memory
2408 * size then this is not a candidate for identity mapping.
2409 */
2410 u64 dma_mask = pdev->dma_mask;
2411
2412 if (pdev->dev.coherent_dma_mask &&
2413 pdev->dev.coherent_dma_mask < dma_mask)
2414 dma_mask = pdev->dev.coherent_dma_mask;
2415
2416 return dma_mask >= dma_get_required_mask(&pdev->dev);
2417 }
6941af28
DW		 2418
2419 return 1;
2420}
2421
071e1374 2422static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2423{
2c2e2c38
FY		 2424 struct pci_dev *pdev = NULL;
2425 int ret;
2426
19943b0e 2427 ret = si_domain_init(hw);
2c2e2c38
FY		 2428 if (ret)
2429 return -EFAULT;
2430
2c2e2c38 2431 for_each_pci_dev(pdev) {
6941af28 2432 if (iommu_should_identity_map(pdev, 1)) {
5fe60f4e 2433 ret = domain_add_dev_info(si_domain, pdev,
eae460b6
MT		 2434 hw ? CONTEXT_TT_PASS_THROUGH :
2435 CONTEXT_TT_MULTI_LEVEL);
2436 if (ret) {
2437 /* device not associated with an iommu */
2438 if (ret == -ENODEV)
2439 continue;
62edf5dc 2440 return ret;
eae460b6
MT		 2441 }
2442 pr_info("IOMMU: %s identity mapping for device %s\n",
2443 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2444 }
2c2e2c38
FY		 2445 }
2446
2447 return 0;
2448}
2449
b779260b 2450static int __init init_dmars(void)
ba395927
KA		 2451{
2452 struct dmar_drhd_unit *drhd;
2453 struct dmar_rmrr_unit *rmrr;
2454 struct pci_dev *pdev;
2455 struct intel_iommu *iommu;
9d783ba0 2456 int i, ret;
2c2e2c38 2457
ba395927
KA		 2458 /*
2459 * for each drhd
2460 * allocate root
2461 * initialize and program root entry to not present
2462 * endfor
2463 */
2464 for_each_drhd_unit(drhd) {
5e0d2a6f 2465 /*
2466 * lock not needed as this is only incremented in the single
2467 * threaded kernel __init code path all other access are read
2468 * only
2469 */
1b198bb0
MT		 2470 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2471 g_num_of_iommus++;
2472 continue;
2473 }
2474 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2475 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2476 }
2477
d9630fe9
WH		 2478 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2479 GFP_KERNEL);
2480 if (!g_iommus) {
2481 printk(KERN_ERR "Allocating global iommu array failed\n");
2482 ret = -ENOMEM;
2483 goto error;
2484 }
2485
80b20dd8 2486 deferred_flush = kzalloc(g_num_of_iommus *
2487 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2488 if (!deferred_flush) {
5e0d2a6f 2489 ret = -ENOMEM;
2490 goto error;
2491 }
2492
5e0d2a6f 2493 for_each_drhd_unit(drhd) {
2494 if (drhd->ignored)
2495 continue;
1886e8a9
SS		 2496
2497 iommu = drhd->iommu;
d9630fe9 2498 g_iommus[iommu->seq_id] = iommu;
ba395927 2499
e61d98d8
SS		 2500 ret = iommu_init_domains(iommu);
2501 if (ret)
2502 goto error;
2503
ba395927
KA		 2504 /*
2505 * TBD:
2506 * we could share the same root & context tables
25985edc 2507 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2508 */
2509 ret = iommu_alloc_root_entry(iommu);
2510 if (ret) {
2511 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2512 goto error;
2513 }
4ed0d3e6 2514 if (!ecap_pass_through(iommu->ecap))
19943b0e 2515 hw_pass_through = 0;
ba395927
KA		 2516 }
2517
1531a6a6
SS		 2518 /*
2519 * Start from the sane iommu hardware state.
2520 */
a77b67d4
YS		 2521 for_each_drhd_unit(drhd) {
2522 if (drhd->ignored)
2523 continue;
2524
2525 iommu = drhd->iommu;
1531a6a6
SS		 2526
2527 /*
2528 * If the queued invalidation is already initialized by us
2529 * (for example, while enabling interrupt-remapping) then
2530 * we got the things already rolling from a sane state.
2531 */
2532 if (iommu->qi)
2533 continue;
2534
2535 /*
2536 * Clear any previous faults.
2537 */
2538 dmar_fault(-1, iommu);
2539 /*
2540 * Disable queued invalidation if supported and already enabled
2541 * before OS handover.
2542 */
2543 dmar_disable_qi(iommu);
2544 }
2545
2546 for_each_drhd_unit(drhd) {
2547 if (drhd->ignored)
2548 continue;
2549
2550 iommu = drhd->iommu;
2551
a77b67d4
YS		 2552 if (dmar_enable_qi(iommu)) {
2553 /*
2554 * Queued Invalidate not enabled, use Register Based
2555 * Invalidate
2556 */
2557 iommu->flush.flush_context = __iommu_flush_context;
2558 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2559 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2560 "invalidation\n",
680a7524 2561 iommu->seq_id,
b4e0f9eb 2562 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2563 } else {
2564 iommu->flush.flush_context = qi_flush_context;
2565 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2566 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2567 "invalidation\n",
680a7524 2568 iommu->seq_id,
b4e0f9eb 2569 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2570 }
2571 }
2572
19943b0e 2573 if (iommu_pass_through)
e0fc7e0b
DW		 2574 iommu_identity_mapping |= IDENTMAP_ALL;
2575
d3f13810 2576#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2577 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2578#endif
e0fc7e0b
DW		 2579
2580 check_tylersburg_isoch();
2581
ba395927 2582 /*
19943b0e
DW		 2583 * If pass through is not set or not enabled, setup context entries for
2584 * identity mappings for rmrr, gfx, and isa and may fall back to static
2585 * identity mapping if iommu_identity_mapping is set.
ba395927 2586 */
19943b0e
DW		 2587 if (iommu_identity_mapping) {
2588 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2589 if (ret) {
19943b0e
DW		 2590 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2591 goto error;
ba395927
KA		 2592 }
2593 }
ba395927 2594 /*
19943b0e
DW		 2595 * For each rmrr
2596 * for each dev attached to rmrr
2597 * do
2598 * locate drhd for dev, alloc domain for dev
2599 * allocate free domain
2600 * allocate page table entries for rmrr
2601 * if context not allocated for bus
2602 * allocate and init context
2603 * set present in root table for this bus
2604 * init context with domain, translation etc
2605 * endfor
2606 * endfor
ba395927 2607 */
19943b0e
DW		 2608 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2609 for_each_rmrr_units(rmrr) {
2610 for (i = 0; i < rmrr->devices_cnt; i++) {
2611 pdev = rmrr->devices[i];
2612 /*
2613 * some BIOS lists non-exist devices in DMAR
2614 * table.
2615 */
2616 if (!pdev)
2617 continue;
2618 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2619 if (ret)
2620 printk(KERN_ERR
2621 "IOMMU: mapping reserved region failed\n");
ba395927 2622 }
4ed0d3e6 2623 }
49a0429e 2624
19943b0e
DW		 2625 iommu_prepare_isa();
2626
ba395927
KA		 2627 /*
2628 * for each drhd
2629 * enable fault log
2630 * global invalidate context cache
2631 * global invalidate iotlb
2632 * enable translation
2633 */
2634 for_each_drhd_unit(drhd) {
51a63e67
JC		 2635 if (drhd->ignored) {
2636 /*
2637 * we always have to disable PMRs or DMA may fail on
2638 * this device
2639 */
2640 if (force_on)
2641 iommu_disable_protect_mem_regions(drhd->iommu);
ba395927 2642 continue;
51a63e67 2643 }
ba395927 2644 iommu = drhd->iommu;
ba395927
KA		 2645
2646 iommu_flush_write_buffer(iommu);
2647
3460a6d9
KA		 2648 ret = dmar_set_interrupt(iommu);
2649 if (ret)
2650 goto error;
2651
ba395927
KA		 2652 iommu_set_root_entry(iommu);
2653
4c25a2c1 2654 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2655 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2656
ba395927
KA		 2657 ret = iommu_enable_translation(iommu);
2658 if (ret)
2659 goto error;
b94996c9
DW		 2660
2661 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2662 }
2663
2664 return 0;
2665error:
2666 for_each_drhd_unit(drhd) {
2667 if (drhd->ignored)
2668 continue;
2669 iommu = drhd->iommu;
2670 free_iommu(iommu);
2671 }
d9630fe9 2672 kfree(g_iommus);
ba395927
KA		 2673 return ret;
2674}
2675
5a5e02a6 2676/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2677static struct iova *intel_alloc_iova(struct device *dev,
2678 struct dmar_domain *domain,
2679 unsigned long nrpages, uint64_t dma_mask)
ba395927 2680{
ba395927 2681 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2682 struct iova *iova = NULL;
ba395927 2683
875764de
DW		 2684 /* Restrict dma_mask to the width that the iommu can handle */
2685 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2686
2687 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2688 /*
2689 * First try to allocate an io virtual address in
284901a9 2690 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2691 * from higher range
ba395927 2692 */
875764de
DW		 2693 iova = alloc_iova(&domain->iovad, nrpages,
2694 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2695 if (iova)
2696 return iova;
2697 }
2698 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2699 if (unlikely(!iova)) {
2700 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2701 nrpages, pci_name(pdev));
f76aec76
KA		 2702 return NULL;
2703 }
2704
2705 return iova;
2706}
2707
147202aa 2708static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2709{
2710 struct dmar_domain *domain;
2711 int ret;
2712
2713 domain = get_domain_for_dev(pdev,
2714 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2715 if (!domain) {
2716 printk(KERN_ERR
2717 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2718 return NULL;
ba395927
KA		 2719 }
2720
2721 /* make sure context mapping is ok */
5331fe6f 2722 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2723 ret = domain_context_mapping(domain, pdev,
2724 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2725 if (ret) {
2726 printk(KERN_ERR
2727 "Domain context map for %s failed",
2728 pci_name(pdev));
4fe05bbc 2729 return NULL;
f76aec76 2730 }
ba395927
KA		 2731 }
2732
f76aec76
KA		 2733 return domain;
2734}
2735
147202aa
DW		 2736static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2737{
2738 struct device_domain_info *info;
2739
2740 /* No lock here, assumes no domain exit in normal case */
2741 info = dev->dev.archdata.iommu;
2742 if (likely(info))
2743 return info->domain;
2744
2745 return __get_valid_domain_for_dev(dev);
2746}
2747
2c2e2c38
FY		 2748static int iommu_dummy(struct pci_dev *pdev)
2749{
2750 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2751}
2752
2753/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2754static int iommu_no_mapping(struct device *dev)
2c2e2c38 2755{
73676832 2756 struct pci_dev *pdev;
2c2e2c38
FY		 2757 int found;
2758
73676832
DW		 2759 if (unlikely(dev->bus != &pci_bus_type))
2760 return 1;
2761
2762 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2763 if (iommu_dummy(pdev))
2764 return 1;
2765
2c2e2c38 2766 if (!iommu_identity_mapping)
1e4c64c4 2767 return 0;
2c2e2c38
FY		 2768
2769 found = identity_mapping(pdev);
2770 if (found) {
6941af28 2771 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2772 return 1;
2773 else {
2774 /*
2775 * 32 bit DMA is removed from si_domain and fall back
2776 * to non-identity mapping.
2777 */
2778 domain_remove_one_dev_info(si_domain, pdev);
2779 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2780 pci_name(pdev));
2781 return 0;
2782 }
2783 } else {
2784 /*
2785 * In case of a detached 64 bit DMA device from vm, the device
2786 * is put into si_domain for identity mapping.
2787 */
6941af28 2788 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2789 int ret;
5fe60f4e
DW		 2790 ret = domain_add_dev_info(si_domain, pdev,
2791 hw_pass_through ?
2792 CONTEXT_TT_PASS_THROUGH :
2793 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2794 if (!ret) {
2795 printk(KERN_INFO "64bit %s uses identity mapping\n",
2796 pci_name(pdev));
2797 return 1;
2798 }
2799 }
2800 }
2801
1e4c64c4 2802 return 0;
2c2e2c38
FY		 2803}
2804
bb9e6d65
FT		 2805static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2806 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2807{
2808 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2809 struct dmar_domain *domain;
5b6985ce 2810 phys_addr_t start_paddr;
f76aec76
KA		 2811 struct iova *iova;
2812 int prot = 0;
6865f0d1 2813 int ret;
8c11e798 2814 struct intel_iommu *iommu;
33041ec0 2815 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2816
2817 BUG_ON(dir == DMA_NONE);
2c2e2c38 2818
73676832 2819 if (iommu_no_mapping(hwdev))
6865f0d1 2820 return paddr;
f76aec76
KA		 2821
2822 domain = get_valid_domain_for_dev(pdev);
2823 if (!domain)
2824 return 0;
2825
8c11e798 2826 iommu = domain_get_iommu(domain);
88cb6a74 2827 size = aligned_nrpages(paddr, size);
f76aec76 2828
c681d0ba 2829 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 2830 if (!iova)
2831 goto error;
2832
ba395927
KA		 2833 /*
2834 * Check if DMAR supports zero-length reads on write only
2835 * mappings..
2836 */
2837 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2838 !cap_zlr(iommu->cap))
ba395927
KA		 2839 prot |= DMA_PTE_READ;
2840 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2841 prot |= DMA_PTE_WRITE;
2842 /*
6865f0d1 2843 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2844 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2845 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2846 * is not a big problem
2847 */
0ab36de2 2848 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2849 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2850 if (ret)
2851 goto error;
2852
1f0ef2aa
DW		 2853 /* it's a non-present to present mapping. Only flush if caching mode */
2854 if (cap_caching_mode(iommu->cap))
82653633 2855 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2856 else
8c11e798 2857 iommu_flush_write_buffer(iommu);
f76aec76 2858
03d6a246
DW		 2859 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2860 start_paddr += paddr & ~PAGE_MASK;
2861 return start_paddr;
ba395927 2862
ba395927 2863error:
f76aec76
KA		 2864 if (iova)
2865 __free_iova(&domain->iovad, iova);
4cf2e75d 2866 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2867 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2868 return 0;
2869}
2870
ffbbef5c
FT		 2871static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2872 unsigned long offset, size_t size,
2873 enum dma_data_direction dir,
2874 struct dma_attrs *attrs)
bb9e6d65 2875{
ffbbef5c
FT		 2876 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2877 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2878}
2879
5e0d2a6f 2880static void flush_unmaps(void)
2881{
80b20dd8 2882 int i, j;
5e0d2a6f 2883
5e0d2a6f 2884 timer_on = 0;
2885
2886 /* just flush them all */
2887 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2888 struct intel_iommu *iommu = g_iommus[i];
2889 if (!iommu)
2890 continue;
c42d9f32 2891
9dd2fe89
YZ		 2892 if (!deferred_flush[i].next)
2893 continue;
2894
78d5f0f5
NA		 2895 /* In caching mode, global flushes turn emulation expensive */
2896 if (!cap_caching_mode(iommu->cap))
2897 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2898 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2899 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2900 unsigned long mask;
2901 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2902 struct dmar_domain *domain = deferred_flush[i].domain[j];
2903
2904 /* On real hardware multiple invalidations are expensive */
2905 if (cap_caching_mode(iommu->cap))
2906 iommu_flush_iotlb_psi(iommu, domain->id,
2907 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2908 else {
2909 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2910 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2911 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2912 }
93a23a72 2913 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2914 }
9dd2fe89 2915 deferred_flush[i].next = 0;
5e0d2a6f 2916 }
2917
5e0d2a6f 2918 list_size = 0;
5e0d2a6f 2919}
2920
2921static void flush_unmaps_timeout(unsigned long data)
2922{
80b20dd8 2923 unsigned long flags;
2924
2925 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2926 flush_unmaps();
80b20dd8 2927 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2928}
2929
2930static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2931{
2932 unsigned long flags;
80b20dd8 2933 int next, iommu_id;
8c11e798 2934 struct intel_iommu *iommu;
5e0d2a6f 2935
2936 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2937 if (list_size == HIGH_WATER_MARK)
2938 flush_unmaps();
2939
8c11e798
WH		 2940 iommu = domain_get_iommu(dom);
2941 iommu_id = iommu->seq_id;
c42d9f32 2942
80b20dd8 2943 next = deferred_flush[iommu_id].next;
2944 deferred_flush[iommu_id].domain[next] = dom;
2945 deferred_flush[iommu_id].iova[next] = iova;
2946 deferred_flush[iommu_id].next++;
5e0d2a6f 2947
2948 if (!timer_on) {
2949 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2950 timer_on = 1;
2951 }
2952 list_size++;
2953 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2954}
2955
ffbbef5c
FT		 2956static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2957 size_t size, enum dma_data_direction dir,
2958 struct dma_attrs *attrs)
ba395927 2959{
ba395927 2960 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2961 struct dmar_domain *domain;
d794dc9b 2962 unsigned long start_pfn, last_pfn;
ba395927 2963 struct iova *iova;
8c11e798 2964 struct intel_iommu *iommu;
ba395927 2965
73676832 2966 if (iommu_no_mapping(dev))
f76aec76 2967 return;
2c2e2c38 2968
ba395927
KA		 2969 domain = find_domain(pdev);
2970 BUG_ON(!domain);
2971
8c11e798
WH		 2972 iommu = domain_get_iommu(domain);
2973
ba395927 2974 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2975 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2976 (unsigned long long)dev_addr))
ba395927 2977 return;
ba395927 2978
d794dc9b
DW		 2979 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2980 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2981
d794dc9b
DW		 2982 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2983 pci_name(pdev), start_pfn, last_pfn);
ba395927 2984
f76aec76 2985 /* clear the whole page */
d794dc9b
DW		 2986 dma_pte_clear_range(domain, start_pfn, last_pfn);
2987
f76aec76 2988 /* free page tables */
d794dc9b
DW		 2989 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2990
5e0d2a6f 2991 if (intel_iommu_strict) {
03d6a246 2992 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2993 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2994 /* free iova */
2995 __free_iova(&domain->iovad, iova);
2996 } else {
2997 add_unmap(domain, iova);
2998 /*
2999 * queue up the release of the unmap to save the 1/6th of the
3000 * cpu used up by the iotlb flush operation...
3001 */
5e0d2a6f 3002 }
ba395927
KA		 3003}
3004
d7ab5c46 3005static void *intel_alloc_coherent(struct device *hwdev, size_t size,
baa676fc
AP		 3006 dma_addr_t *dma_handle, gfp_t flags,
3007 struct dma_attrs *attrs)
ba395927
KA		 3008{
3009 void *vaddr;
3010 int order;
3011
5b6985ce 3012 size = PAGE_ALIGN(size);
ba395927 3013 order = get_order(size);
e8bb910d
AW		 3014
3015 if (!iommu_no_mapping(hwdev))
3016 flags &= ~(GFP_DMA | GFP_DMA32);
3017 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
3018 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
3019 flags |= GFP_DMA;
3020 else
3021 flags |= GFP_DMA32;
3022 }
ba395927
KA		 3023
3024 vaddr = (void *)__get_free_pages(flags, order);
3025 if (!vaddr)
3026 return NULL;
3027 memset(vaddr, 0, size);
3028
bb9e6d65
FT		 3029 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
3030 DMA_BIDIRECTIONAL,
3031 hwdev->coherent_dma_mask);
ba395927
KA		 3032 if (*dma_handle)
3033 return vaddr;
3034 free_pages((unsigned long)vaddr, order);
3035 return NULL;
3036}
3037
d7ab5c46 3038static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
baa676fc 3039 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3040{
3041 int order;
3042
5b6985ce 3043 size = PAGE_ALIGN(size);
ba395927
KA		 3044 order = get_order(size);
3045
0db9b7ae 3046 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 3047 free_pages((unsigned long)vaddr, order);
3048}
3049
d7ab5c46
FT		 3050static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3051 int nelems, enum dma_data_direction dir,
3052 struct dma_attrs *attrs)
ba395927 3053{
ba395927
KA		 3054 struct pci_dev *pdev = to_pci_dev(hwdev);
3055 struct dmar_domain *domain;
d794dc9b 3056 unsigned long start_pfn, last_pfn;
f76aec76 3057 struct iova *iova;
8c11e798 3058 struct intel_iommu *iommu;
ba395927 3059
73676832 3060 if (iommu_no_mapping(hwdev))
ba395927
KA		 3061 return;
3062
3063 domain = find_domain(pdev);
8c11e798
WH		 3064 BUG_ON(!domain);
3065
3066 iommu = domain_get_iommu(domain);
ba395927 3067
c03ab37c 3068 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3069 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3070 (unsigned long long)sglist[0].dma_address))
f76aec76 3071 return;
f76aec76 3072
d794dc9b
DW		 3073 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3074 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 3075
3076 /* clear the whole page */
d794dc9b
DW		 3077 dma_pte_clear_range(domain, start_pfn, last_pfn);
3078
f76aec76 3079 /* free page tables */
d794dc9b 3080 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 3081
acea0018
DW		 3082 if (intel_iommu_strict) {
3083 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 3084 last_pfn - start_pfn + 1, 0);
acea0018
DW		 3085 /* free iova */
3086 __free_iova(&domain->iovad, iova);
3087 } else {
3088 add_unmap(domain, iova);
3089 /*
3090 * queue up the release of the unmap to save the 1/6th of the
3091 * cpu used up by the iotlb flush operation...
3092 */
3093 }
ba395927
KA		 3094}
3095
ba395927 3096static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3097 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3098{
3099 int i;
c03ab37c 3100 struct scatterlist *sg;
ba395927 3101
c03ab37c 3102 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3103 BUG_ON(!sg_page(sg));
4cf2e75d 3104 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3105 sg->dma_length = sg->length;
ba395927
KA		 3106 }
3107 return nelems;
3108}
3109
d7ab5c46
FT		 3110static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3111 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3112{
ba395927 3113 int i;
ba395927
KA		 3114 struct pci_dev *pdev = to_pci_dev(hwdev);
3115 struct dmar_domain *domain;
f76aec76
KA		 3116 size_t size = 0;
3117 int prot = 0;
f76aec76
KA		 3118 struct iova *iova = NULL;
3119 int ret;
c03ab37c 3120 struct scatterlist *sg;
b536d24d 3121 unsigned long start_vpfn;
8c11e798 3122 struct intel_iommu *iommu;
ba395927
KA		 3123
3124 BUG_ON(dir == DMA_NONE);
73676832 3125 if (iommu_no_mapping(hwdev))
c03ab37c 3126 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3127
f76aec76
KA		 3128 domain = get_valid_domain_for_dev(pdev);
3129 if (!domain)
3130 return 0;
3131
8c11e798
WH		 3132 iommu = domain_get_iommu(domain);
3133
b536d24d 3134 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3135 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3136
5a5e02a6
DW		 3137 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3138 pdev->dma_mask);
f76aec76 3139 if (!iova) {
c03ab37c 3140 sglist->dma_length = 0;
f76aec76
KA		 3141 return 0;
3142 }
3143
3144 /*
3145 * Check if DMAR supports zero-length reads on write only
3146 * mappings..
3147 */
3148 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3149 !cap_zlr(iommu->cap))
f76aec76
KA		 3150 prot |= DMA_PTE_READ;
3151 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3152 prot |= DMA_PTE_WRITE;
3153
b536d24d 3154 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3155
f532959b 3156 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3157 if (unlikely(ret)) {
3158 /* clear the page */
3159 dma_pte_clear_range(domain, start_vpfn,
3160 start_vpfn + size - 1);
3161 /* free page tables */
3162 dma_pte_free_pagetable(domain, start_vpfn,
3163 start_vpfn + size - 1);
3164 /* free iova */
3165 __free_iova(&domain->iovad, iova);
3166 return 0;
ba395927
KA		 3167 }
3168
1f0ef2aa
DW		 3169 /* it's a non-present to present mapping. Only flush if caching mode */
3170 if (cap_caching_mode(iommu->cap))
82653633 3171 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 3172 else
8c11e798 3173 iommu_flush_write_buffer(iommu);
1f0ef2aa 3174
ba395927
KA		 3175 return nelems;
3176}
3177
dfb805e8
FT		 3178static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3179{
3180 return !dma_addr;
3181}
3182
160c1d8e 3183struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3184 .alloc = intel_alloc_coherent,
3185 .free = intel_free_coherent,
ba395927
KA		 3186 .map_sg = intel_map_sg,
3187 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3188 .map_page = intel_map_page,
3189 .unmap_page = intel_unmap_page,
dfb805e8 3190 .mapping_error = intel_mapping_error,
ba395927
KA		 3191};
3192
3193static inline int iommu_domain_cache_init(void)
3194{
3195 int ret = 0;
3196
3197 iommu_domain_cache = kmem_cache_create("iommu_domain",
3198 sizeof(struct dmar_domain),
3199 0,
3200 SLAB_HWCACHE_ALIGN,
3201
3202 NULL);
3203 if (!iommu_domain_cache) {
3204 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3205 ret = -ENOMEM;
3206 }
3207
3208 return ret;
3209}
3210
3211static inline int iommu_devinfo_cache_init(void)
3212{
3213 int ret = 0;
3214
3215 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3216 sizeof(struct device_domain_info),
3217 0,
3218 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3219 NULL);
3220 if (!iommu_devinfo_cache) {
3221 printk(KERN_ERR "Couldn't create devinfo cache\n");
3222 ret = -ENOMEM;
3223 }
3224
3225 return ret;
3226}
3227
3228static inline int iommu_iova_cache_init(void)
3229{
3230 int ret = 0;
3231
3232 iommu_iova_cache = kmem_cache_create("iommu_iova",
3233 sizeof(struct iova),
3234 0,
3235 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3236 NULL);
3237 if (!iommu_iova_cache) {
3238 printk(KERN_ERR "Couldn't create iova cache\n");
3239 ret = -ENOMEM;
3240 }
3241
3242 return ret;
3243}
3244
3245static int __init iommu_init_mempool(void)
3246{
3247 int ret;
3248 ret = iommu_iova_cache_init();
3249 if (ret)
3250 return ret;
3251
3252 ret = iommu_domain_cache_init();
3253 if (ret)
3254 goto domain_error;
3255
3256 ret = iommu_devinfo_cache_init();
3257 if (!ret)
3258 return ret;
3259
3260 kmem_cache_destroy(iommu_domain_cache);
3261domain_error:
3262 kmem_cache_destroy(iommu_iova_cache);
3263
3264 return -ENOMEM;
3265}
3266
3267static void __init iommu_exit_mempool(void)
3268{
3269 kmem_cache_destroy(iommu_devinfo_cache);
3270 kmem_cache_destroy(iommu_domain_cache);
3271 kmem_cache_destroy(iommu_iova_cache);
3272
3273}
3274
556ab45f
DW		 3275static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3276{
3277 struct dmar_drhd_unit *drhd;
3278 u32 vtbar;
3279 int rc;
3280
3281 /* We know that this device on this chipset has its own IOMMU.
3282 * If we find it under a different IOMMU, then the BIOS is lying
3283 * to us. Hope that the IOMMU for this device is actually
3284 * disabled, and it needs no translation...
3285 */
3286 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3287 if (rc) {
3288 /* "can't" happen */
3289 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3290 return;
3291 }
3292 vtbar &= 0xffff0000;
3293
3294 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3295 drhd = dmar_find_matched_drhd_unit(pdev);
3296 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3297 TAINT_FIRMWARE_WORKAROUND,
3298 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3299 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3300}
3301DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3302
ba395927
KA		 3303static void __init init_no_remapping_devices(void)
3304{
3305 struct dmar_drhd_unit *drhd;
3306
3307 for_each_drhd_unit(drhd) {
3308 if (!drhd->include_all) {
3309 int i;
3310 for (i = 0; i < drhd->devices_cnt; i++)
3311 if (drhd->devices[i] != NULL)
3312 break;
3313 /* ignore DMAR unit if no pci devices exist */
3314 if (i == drhd->devices_cnt)
3315 drhd->ignored = 1;
3316 }
3317 }
3318
ba395927
KA		 3319 for_each_drhd_unit(drhd) {
3320 int i;
3321 if (drhd->ignored || drhd->include_all)
3322 continue;
3323
3324 for (i = 0; i < drhd->devices_cnt; i++)
3325 if (drhd->devices[i] &&
c0771df8 3326 !IS_GFX_DEVICE(drhd->devices[i]))
ba395927
KA		 3327 break;
3328
3329 if (i < drhd->devices_cnt)
3330 continue;
3331
c0771df8
DW		 3332 /* This IOMMU has *only* gfx devices. Either bypass it or
3333 set the gfx_mapped flag, as appropriate */
3334 if (dmar_map_gfx) {
3335 intel_iommu_gfx_mapped = 1;
3336 } else {
3337 drhd->ignored = 1;
3338 for (i = 0; i < drhd->devices_cnt; i++) {
3339 if (!drhd->devices[i])
3340 continue;
3341 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3342 }
ba395927
KA		 3343 }
3344 }
3345}
3346
f59c7b69
FY		 3347#ifdef CONFIG_SUSPEND
3348static int init_iommu_hw(void)
3349{
3350 struct dmar_drhd_unit *drhd;
3351 struct intel_iommu *iommu = NULL;
3352
3353 for_each_active_iommu(iommu, drhd)
3354 if (iommu->qi)
3355 dmar_reenable_qi(iommu);
3356
b779260b
JC		 3357 for_each_iommu(iommu, drhd) {
3358 if (drhd->ignored) {
3359 /*
3360 * we always have to disable PMRs or DMA may fail on
3361 * this device
3362 */
3363 if (force_on)
3364 iommu_disable_protect_mem_regions(iommu);
3365 continue;
3366 }
3367
f59c7b69
FY		 3368 iommu_flush_write_buffer(iommu);
3369
3370 iommu_set_root_entry(iommu);
3371
3372 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3373 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3374 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3375 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3376 if (iommu_enable_translation(iommu))
3377 return 1;
b94996c9 3378 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3379 }
3380
3381 return 0;
3382}
3383
3384static void iommu_flush_all(void)
3385{
3386 struct dmar_drhd_unit *drhd;
3387 struct intel_iommu *iommu;
3388
3389 for_each_active_iommu(iommu, drhd) {
3390 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3391 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3392 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3393 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3394 }
3395}
3396
134fac3f 3397static int iommu_suspend(void)
f59c7b69
FY		 3398{
3399 struct dmar_drhd_unit *drhd;
3400 struct intel_iommu *iommu = NULL;
3401 unsigned long flag;
3402
3403 for_each_active_iommu(iommu, drhd) {
3404 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3405 GFP_ATOMIC);
3406 if (!iommu->iommu_state)
3407 goto nomem;
3408 }
3409
3410 iommu_flush_all();
3411
3412 for_each_active_iommu(iommu, drhd) {
3413 iommu_disable_translation(iommu);
3414
1f5b3c3f 3415 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3416
3417 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3418 readl(iommu->reg + DMAR_FECTL_REG);
3419 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3420 readl(iommu->reg + DMAR_FEDATA_REG);
3421 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3422 readl(iommu->reg + DMAR_FEADDR_REG);
3423 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3424 readl(iommu->reg + DMAR_FEUADDR_REG);
3425
1f5b3c3f 3426 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3427 }
3428 return 0;
3429
3430nomem:
3431 for_each_active_iommu(iommu, drhd)
3432 kfree(iommu->iommu_state);
3433
3434 return -ENOMEM;
3435}
3436
134fac3f 3437static void iommu_resume(void)
f59c7b69
FY		 3438{
3439 struct dmar_drhd_unit *drhd;
3440 struct intel_iommu *iommu = NULL;
3441 unsigned long flag;
3442
3443 if (init_iommu_hw()) {
b779260b
JC		 3444 if (force_on)
3445 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3446 else
3447 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3448 return;
f59c7b69
FY		 3449 }
3450
3451 for_each_active_iommu(iommu, drhd) {
3452
1f5b3c3f 3453 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3454
3455 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3456 iommu->reg + DMAR_FECTL_REG);
3457 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3458 iommu->reg + DMAR_FEDATA_REG);
3459 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3460 iommu->reg + DMAR_FEADDR_REG);
3461 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3462 iommu->reg + DMAR_FEUADDR_REG);
3463
1f5b3c3f 3464 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3465 }
3466
3467 for_each_active_iommu(iommu, drhd)
3468 kfree(iommu->iommu_state);
f59c7b69
FY		 3469}
3470
134fac3f 3471static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3472 .resume = iommu_resume,
3473 .suspend = iommu_suspend,
3474};
3475
134fac3f 3476static void __init init_iommu_pm_ops(void)
f59c7b69 3477{
134fac3f 3478 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3479}
3480
3481#else
99592ba4 3482static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3483#endif /* CONFIG_PM */
3484
318fe7df
SS		 3485LIST_HEAD(dmar_rmrr_units);
3486
3487static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3488{
3489 list_add(&rmrr->list, &dmar_rmrr_units);
3490}
3491
3492
3493int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3494{
3495 struct acpi_dmar_reserved_memory *rmrr;
3496 struct dmar_rmrr_unit *rmrru;
3497
3498 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3499 if (!rmrru)
3500 return -ENOMEM;
3501
3502 rmrru->hdr = header;
3503 rmrr = (struct acpi_dmar_reserved_memory *)header;
3504 rmrru->base_address = rmrr->base_address;
3505 rmrru->end_address = rmrr->end_address;
3506
3507 dmar_register_rmrr_unit(rmrru);
3508 return 0;
3509}
3510
3511static int __init
3512rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3513{
3514 struct acpi_dmar_reserved_memory *rmrr;
3515 int ret;
3516
3517 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3518 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3519 ((void *)rmrr) + rmrr->header.length,
3520 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3521
3522 if (ret || (rmrru->devices_cnt == 0)) {
3523 list_del(&rmrru->list);
3524 kfree(rmrru);
3525 }
3526 return ret;
3527}
3528
3529static LIST_HEAD(dmar_atsr_units);
3530
3531int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3532{
3533 struct acpi_dmar_atsr *atsr;
3534 struct dmar_atsr_unit *atsru;
3535
3536 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3537 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3538 if (!atsru)
3539 return -ENOMEM;
3540
3541 atsru->hdr = hdr;
3542 atsru->include_all = atsr->flags & 0x1;
3543
3544 list_add(&atsru->list, &dmar_atsr_units);
3545
3546 return 0;
3547}
3548
3549static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3550{
3551 int rc;
3552 struct acpi_dmar_atsr *atsr;
3553
3554 if (atsru->include_all)
3555 return 0;
3556
3557 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3558 rc = dmar_parse_dev_scope((void *)(atsr + 1),
3559 (void *)atsr + atsr->header.length,
3560 &atsru->devices_cnt, &atsru->devices,
3561 atsr->segment);
3562 if (rc || !atsru->devices_cnt) {
3563 list_del(&atsru->list);
3564 kfree(atsru);
3565 }
3566
3567 return rc;
3568}
3569
3570int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3571{
3572 int i;
3573 struct pci_bus *bus;
3574 struct acpi_dmar_atsr *atsr;
3575 struct dmar_atsr_unit *atsru;
3576
3577 dev = pci_physfn(dev);
3578
3579 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3580 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3581 if (atsr->segment == pci_domain_nr(dev->bus))
3582 goto found;
3583 }
3584
3585 return 0;
3586
3587found:
3588 for (bus = dev->bus; bus; bus = bus->parent) {
3589 struct pci_dev *bridge = bus->self;
3590
3591 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3592 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df
SS		 3593 return 0;
3594
62f87c0e 3595 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
318fe7df
SS		 3596 for (i = 0; i < atsru->devices_cnt; i++)
3597 if (atsru->devices[i] == bridge)
3598 return 1;
3599 break;
3600 }
3601 }
3602
3603 if (atsru->include_all)
3604 return 1;
3605
3606 return 0;
3607}
3608
c8f369ab 3609int __init dmar_parse_rmrr_atsr_dev(void)
318fe7df
SS		 3610{
3611 struct dmar_rmrr_unit *rmrr, *rmrr_n;
3612 struct dmar_atsr_unit *atsr, *atsr_n;
3613 int ret = 0;
3614
3615 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3616 ret = rmrr_parse_dev(rmrr);
3617 if (ret)
3618 return ret;
3619 }
3620
3621 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3622 ret = atsr_parse_dev(atsr);
3623 if (ret)
3624 return ret;
3625 }
3626
3627 return ret;
3628}
3629
99dcaded
FY		 3630/*
3631 * Here we only respond to action of unbound device from driver.
3632 *
3633 * Added device is not attached to its DMAR domain here yet. That will happen
3634 * when mapping the device to iova.
3635 */
3636static int device_notifier(struct notifier_block *nb,
3637 unsigned long action, void *data)
3638{
3639 struct device *dev = data;
3640 struct pci_dev *pdev = to_pci_dev(dev);
3641 struct dmar_domain *domain;
3642
44cd613c
DW		 3643 if (iommu_no_mapping(dev))
3644 return 0;
3645
99dcaded
FY		 3646 domain = find_domain(pdev);
3647 if (!domain)
3648 return 0;
3649
a97590e5 3650 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3651 domain_remove_one_dev_info(domain, pdev);
3652
a97590e5
AW		 3653 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3654 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3655 list_empty(&domain->devices))
3656 domain_exit(domain);
3657 }
3658
99dcaded
FY		 3659 return 0;
3660}
3661
3662static struct notifier_block device_nb = {
3663 .notifier_call = device_notifier,
3664};
3665
ba395927
KA		 3666int __init intel_iommu_init(void)
3667{
3668 int ret = 0;
3a93c841 3669 struct dmar_drhd_unit *drhd;
ba395927 3670
a59b50e9
JC		 3671 /* VT-d is required for a TXT/tboot launch, so enforce that */
3672 force_on = tboot_force_iommu();
3673
3674 if (dmar_table_init()) {
3675 if (force_on)
3676 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3677 return -ENODEV;
a59b50e9 3678 }
ba395927 3679
3a93c841
TI		 3680 /*
3681 * Disable translation if already enabled prior to OS handover.
3682 */
3683 for_each_drhd_unit(drhd) {
3684 struct intel_iommu *iommu;
3685
3686 if (drhd->ignored)
3687 continue;
3688
3689 iommu = drhd->iommu;
3690 if (iommu->gcmd & DMA_GCMD_TE)
3691 iommu_disable_translation(iommu);
3692 }
3693
c2c7286a 3694 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3695 if (force_on)
3696 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3697 return -ENODEV;
a59b50e9 3698 }
1886e8a9 3699
75f1cdf1 3700 if (no_iommu || dmar_disabled)
2ae21010
SS		 3701 return -ENODEV;
3702
51a63e67
JC		 3703 if (iommu_init_mempool()) {
3704 if (force_on)
3705 panic("tboot: Failed to initialize iommu memory\n");
3706 return -ENODEV;
3707 }
3708
318fe7df
SS		 3709 if (list_empty(&dmar_rmrr_units))
3710 printk(KERN_INFO "DMAR: No RMRR found\n");
3711
3712 if (list_empty(&dmar_atsr_units))
3713 printk(KERN_INFO "DMAR: No ATSR found\n");
3714
51a63e67
JC		 3715 if (dmar_init_reserved_ranges()) {
3716 if (force_on)
3717 panic("tboot: Failed to reserve iommu ranges\n");
3718 return -ENODEV;
3719 }
ba395927
KA		 3720
3721 init_no_remapping_devices();
3722
b779260b 3723 ret = init_dmars();
ba395927 3724 if (ret) {
a59b50e9
JC		 3725 if (force_on)
3726 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3727 printk(KERN_ERR "IOMMU: dmar init failed\n");
3728 put_iova_domain(&reserved_iova_list);
3729 iommu_exit_mempool();
3730 return ret;
3731 }
3732 printk(KERN_INFO
3733 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3734
5e0d2a6f 3735 init_timer(&unmap_timer);
75f1cdf1
FT		 3736#ifdef CONFIG_SWIOTLB
3737 swiotlb = 0;
3738#endif
19943b0e 3739 dma_ops = &intel_dma_ops;
4ed0d3e6 3740
134fac3f 3741 init_iommu_pm_ops();
a8bcbb0d 3742
4236d97d 3743 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
a8bcbb0d 3744
99dcaded
FY		 3745 bus_register_notifier(&pci_bus_type, &device_nb);
3746
8bc1f85c
ED		 3747 intel_iommu_enabled = 1;
3748
ba395927
KA		 3749 return 0;
3750}
e820482c 3751
3199aa6b
HW		 3752static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3753 struct pci_dev *pdev)
3754{
3755 struct pci_dev *tmp, *parent;
3756
3757 if (!iommu || !pdev)
3758 return;
3759
3760 /* dependent device detach */
3761 tmp = pci_find_upstream_pcie_bridge(pdev);
3762 /* Secondary interface's bus number and devfn 0 */
3763 if (tmp) {
3764 parent = pdev->bus->self;
3765 while (parent != tmp) {
3766 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3767 parent->devfn);
3199aa6b
HW		 3768 parent = parent->bus->self;
3769 }
45e829ea 3770 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3771 iommu_detach_dev(iommu,
3772 tmp->subordinate->number, 0);
3773 else /* this is a legacy PCI bridge */
276dbf99
DW		 3774 iommu_detach_dev(iommu, tmp->bus->number,
3775 tmp->devfn);
3199aa6b
HW		 3776 }
3777}
3778
2c2e2c38 3779static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3780 struct pci_dev *pdev)
3781{
3782 struct device_domain_info *info;
3783 struct intel_iommu *iommu;
3784 unsigned long flags;
3785 int found = 0;
3786 struct list_head *entry, *tmp;
3787
276dbf99
DW		 3788 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3789 pdev->devfn);
c7151a8d
WH		 3790 if (!iommu)
3791 return;
3792
3793 spin_lock_irqsave(&device_domain_lock, flags);
3794 list_for_each_safe(entry, tmp, &domain->devices) {
3795 info = list_entry(entry, struct device_domain_info, link);
8519dc44
MH		 3796 if (info->segment == pci_domain_nr(pdev->bus) &&
3797 info->bus == pdev->bus->number &&
c7151a8d 3798 info->devfn == pdev->devfn) {
109b9b04 3799 unlink_domain_info(info);
c7151a8d
WH		 3800 spin_unlock_irqrestore(&device_domain_lock, flags);
3801
93a23a72 3802 iommu_disable_dev_iotlb(info);
c7151a8d 3803 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3804 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3805 free_devinfo_mem(info);
3806
3807 spin_lock_irqsave(&device_domain_lock, flags);
3808
3809 if (found)
3810 break;
3811 else
3812 continue;
3813 }
3814
3815 /* if there is no other devices under the same iommu
3816 * owned by this domain, clear this iommu in iommu_bmp
3817 * update iommu count and coherency
3818 */
276dbf99
DW		 3819 if (iommu == device_to_iommu(info->segment, info->bus,
3820 info->devfn))
c7151a8d
WH		 3821 found = 1;
3822 }
3823
3e7abe25
RD		 3824 spin_unlock_irqrestore(&device_domain_lock, flags);
3825
c7151a8d
WH		 3826 if (found == 0) {
3827 unsigned long tmp_flags;
3828 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 3829 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 3830 domain->iommu_count--;
58c610bd 3831 domain_update_iommu_cap(domain);
c7151a8d 3832 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 3833
9b4554b2
AW		 3834 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3835 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3836 spin_lock_irqsave(&iommu->lock, tmp_flags);
3837 clear_bit(domain->id, iommu->domain_ids);
3838 iommu->domains[domain->id] = NULL;
3839 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3840 }
c7151a8d 3841 }
c7151a8d
WH		 3842}
3843
3844static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3845{
3846 struct device_domain_info *info;
3847 struct intel_iommu *iommu;
3848 unsigned long flags1, flags2;
3849
3850 spin_lock_irqsave(&device_domain_lock, flags1);
3851 while (!list_empty(&domain->devices)) {
3852 info = list_entry(domain->devices.next,
3853 struct device_domain_info, link);
109b9b04 3854 unlink_domain_info(info);
c7151a8d
WH		 3855 spin_unlock_irqrestore(&device_domain_lock, flags1);
3856
93a23a72 3857 iommu_disable_dev_iotlb(info);
276dbf99 3858 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3859 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3860 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3861
3862 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3863 * and capabilities
c7151a8d
WH		 3864 */
3865 spin_lock_irqsave(&domain->iommu_lock, flags2);
3866 if (test_and_clear_bit(iommu->seq_id,
1b198bb0 3867 domain->iommu_bmp)) {
c7151a8d 3868 domain->iommu_count--;
58c610bd 3869 domain_update_iommu_cap(domain);
c7151a8d
WH		 3870 }
3871 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3872
3873 free_devinfo_mem(info);
3874 spin_lock_irqsave(&device_domain_lock, flags1);
3875 }
3876 spin_unlock_irqrestore(&device_domain_lock, flags1);
3877}
3878
5e98c4b1
WH		 3879/* domain id for virtual machine, it won't be set in context */
3880static unsigned long vm_domid;
3881
3882static struct dmar_domain *iommu_alloc_vm_domain(void)
3883{
3884 struct dmar_domain *domain;
3885
3886 domain = alloc_domain_mem();
3887 if (!domain)
3888 return NULL;
3889
3890 domain->id = vm_domid++;
4c923d47 3891 domain->nid = -1;
1b198bb0 3892 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
5e98c4b1
WH		 3893 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3894
3895 return domain;
3896}
3897
2c2e2c38 3898static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3899{
3900 int adjust_width;
3901
3902 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3903 spin_lock_init(&domain->iommu_lock);
3904
3905 domain_reserve_special_ranges(domain);
3906
3907 /* calculate AGAW */
3908 domain->gaw = guest_width;
3909 adjust_width = guestwidth_to_adjustwidth(guest_width);
3910 domain->agaw = width_to_agaw(adjust_width);
3911
3912 INIT_LIST_HEAD(&domain->devices);
3913
3914 domain->iommu_count = 0;
3915 domain->iommu_coherency = 0;
c5b15255 3916 domain->iommu_snooping = 0;
6dd9a7c7 3917 domain->iommu_superpage = 0;
fe40f1e0 3918 domain->max_addr = 0;
4c923d47 3919 domain->nid = -1;
5e98c4b1
WH		 3920
3921 /* always allocate the top pgd */
4c923d47 3922 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3923 if (!domain->pgd)
3924 return -ENOMEM;
3925 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3926 return 0;
3927}
3928
3929static void iommu_free_vm_domain(struct dmar_domain *domain)
3930{
3931 unsigned long flags;
3932 struct dmar_drhd_unit *drhd;
3933 struct intel_iommu *iommu;
3934 unsigned long i;
3935 unsigned long ndomains;
3936
3937 for_each_drhd_unit(drhd) {
3938 if (drhd->ignored)
3939 continue;
3940 iommu = drhd->iommu;
3941
3942 ndomains = cap_ndoms(iommu->cap);
a45946ab 3943 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3944 if (iommu->domains[i] == domain) {
3945 spin_lock_irqsave(&iommu->lock, flags);
3946 clear_bit(i, iommu->domain_ids);
3947 iommu->domains[i] = NULL;
3948 spin_unlock_irqrestore(&iommu->lock, flags);
3949 break;
3950 }
5e98c4b1
WH		 3951 }
3952 }
3953}
3954
3955static void vm_domain_exit(struct dmar_domain *domain)
3956{
5e98c4b1
WH		 3957 /* Domain 0 is reserved, so dont process it */
3958 if (!domain)
3959 return;
3960
3961 vm_domain_remove_all_dev_info(domain);
3962 /* destroy iovas */
3963 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3964
3965 /* clear ptes */
595badf5 3966 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3967
3968 /* free page tables */
d794dc9b 3969 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3970
3971 iommu_free_vm_domain(domain);
3972 free_domain_mem(domain);
3973}
3974
5d450806 3975static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3976{
5d450806 3977 struct dmar_domain *dmar_domain;
38717946 3978
5d450806
JR		 3979 dmar_domain = iommu_alloc_vm_domain();
3980 if (!dmar_domain) {
38717946 3981 printk(KERN_ERR
5d450806
JR		 3982 "intel_iommu_domain_init: dmar_domain == NULL\n");
3983 return -ENOMEM;
38717946 3984 }
2c2e2c38 3985 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3986 printk(KERN_ERR
5d450806
JR		 3987 "intel_iommu_domain_init() failed\n");
3988 vm_domain_exit(dmar_domain);
3989 return -ENOMEM;
38717946 3990 }
8140a95d 3991 domain_update_iommu_cap(dmar_domain);
5d450806 3992 domain->priv = dmar_domain;
faa3d6f5 3993
8a0e715b
JR		 3994 domain->geometry.aperture_start = 0;
3995 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3996 domain->geometry.force_aperture = true;
3997
5d450806 3998 return 0;
38717946 3999}
38717946 4000
5d450806 4001static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4002{
5d450806
JR		 4003 struct dmar_domain *dmar_domain = domain->priv;
4004
4005 domain->priv = NULL;
4006 vm_domain_exit(dmar_domain);
38717946 4007}
38717946 4008
4c5478c9
JR		 4009static int intel_iommu_attach_device(struct iommu_domain *domain,
4010 struct device *dev)
38717946 4011{
4c5478c9
JR		 4012 struct dmar_domain *dmar_domain = domain->priv;
4013 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 4014 struct intel_iommu *iommu;
4015 int addr_width;
faa3d6f5
WH		 4016
4017 /* normally pdev is not mapped */
4018 if (unlikely(domain_context_mapped(pdev))) {
4019 struct dmar_domain *old_domain;
4020
4021 old_domain = find_domain(pdev);
4022 if (old_domain) {
2c2e2c38
FY		 4023 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4024 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4025 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 4026 else
4027 domain_remove_dev_info(old_domain);
4028 }
4029 }
4030
276dbf99
DW		 4031 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4032 pdev->devfn);
fe40f1e0
WH		 4033 if (!iommu)
4034 return -ENODEV;
4035
4036 /* check if this iommu agaw is sufficient for max mapped address */
4037 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4038 if (addr_width > cap_mgaw(iommu->cap))
4039 addr_width = cap_mgaw(iommu->cap);
4040
4041 if (dmar_domain->max_addr > (1LL << addr_width)) {
4042 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4043 "sufficient for the mapped address (%llx)\n",
a99c47a2 4044 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4045 return -EFAULT;
4046 }
a99c47a2
TL		 4047 dmar_domain->gaw = addr_width;
4048
4049 /*
4050 * Knock out extra levels of page tables if necessary
4051 */
4052 while (iommu->agaw < dmar_domain->agaw) {
4053 struct dma_pte *pte;
4054
4055 pte = dmar_domain->pgd;
4056 if (dma_pte_present(pte)) {
25cbff16
SY		 4057 dmar_domain->pgd = (struct dma_pte *)
4058 phys_to_virt(dma_pte_addr(pte));
7a661013 4059 free_pgtable_page(pte);
a99c47a2
TL		 4060 }
4061 dmar_domain->agaw--;
4062 }
fe40f1e0 4063
5fe60f4e 4064 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 4065}
38717946 4066
4c5478c9
JR		 4067static void intel_iommu_detach_device(struct iommu_domain *domain,
4068 struct device *dev)
38717946 4069{
4c5478c9
JR		 4070 struct dmar_domain *dmar_domain = domain->priv;
4071 struct pci_dev *pdev = to_pci_dev(dev);
4072
2c2e2c38 4073 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 4074}
c7151a8d 4075
b146a1c9
JR		 4076static int intel_iommu_map(struct iommu_domain *domain,
4077 unsigned long iova, phys_addr_t hpa,
5009065d 4078 size_t size, int iommu_prot)
faa3d6f5 4079{
dde57a21 4080 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4081 u64 max_addr;
dde57a21 4082 int prot = 0;
faa3d6f5 4083 int ret;
fe40f1e0 4084
dde57a21
JR		 4085 if (iommu_prot & IOMMU_READ)
4086 prot |= DMA_PTE_READ;
4087 if (iommu_prot & IOMMU_WRITE)
4088 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4089 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4090 prot |= DMA_PTE_SNP;
dde57a21 4091
163cc52c 4092 max_addr = iova + size;
dde57a21 4093 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4094 u64 end;
4095
4096 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4097 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4098 if (end < max_addr) {
8954da1f 4099 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4100 "sufficient for the mapped address (%llx)\n",
8954da1f 4101 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4102 return -EFAULT;
4103 }
dde57a21 4104 dmar_domain->max_addr = max_addr;
fe40f1e0 4105 }
ad051221
DW		 4106 /* Round up size to next multiple of PAGE_SIZE, if it and
4107 the low bits of hpa would take us onto the next page */
88cb6a74 4108 size = aligned_nrpages(hpa, size);
ad051221
DW		 4109 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4110 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4111 return ret;
38717946 4112}
38717946 4113
5009065d
OBC		 4114static size_t intel_iommu_unmap(struct iommu_domain *domain,
4115 unsigned long iova, size_t size)
38717946 4116{
dde57a21 4117 struct dmar_domain *dmar_domain = domain->priv;
292827cb 4118 int order;
4b99d352 4119
292827cb 4120 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
163cc52c 4121 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 4122
163cc52c
DW		 4123 if (dmar_domain->max_addr == iova + size)
4124 dmar_domain->max_addr = iova;
b146a1c9 4125
5009065d 4126 return PAGE_SIZE << order;
38717946 4127}
38717946 4128
d14d6577 4129static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4130 dma_addr_t iova)
38717946 4131{
d14d6577 4132 struct dmar_domain *dmar_domain = domain->priv;
38717946 4133 struct dma_pte *pte;
faa3d6f5 4134 u64 phys = 0;
38717946 4135
6dd9a7c7 4136 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
38717946 4137 if (pte)
faa3d6f5 4138 phys = dma_pte_addr(pte);
38717946 4139
faa3d6f5 4140 return phys;
38717946 4141}
a8bcbb0d 4142
dbb9fd86
SY		 4143static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4144 unsigned long cap)
4145{
4146 struct dmar_domain *dmar_domain = domain->priv;
4147
4148 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4149 return dmar_domain->iommu_snooping;
323f99cb 4150 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4151 return irq_remapping_enabled;
dbb9fd86
SY		 4152
4153 return 0;
4154}
4155
783f157b 4156#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
70ae6f0d 4157
abdfdde2
AW		 4158static int intel_iommu_add_device(struct device *dev)
4159{
4160 struct pci_dev *pdev = to_pci_dev(dev);
3da4af0a 4161 struct pci_dev *bridge, *dma_pdev = NULL;
abdfdde2
AW		 4162 struct iommu_group *group;
4163 int ret;
70ae6f0d 4164
abdfdde2
AW		 4165 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4166 pdev->bus->number, pdev->devfn))
70ae6f0d
AW		 4167 return -ENODEV;
4168
4169 bridge = pci_find_upstream_pcie_bridge(pdev);
4170 if (bridge) {
abdfdde2
AW		 4171 if (pci_is_pcie(bridge))
4172 dma_pdev = pci_get_domain_bus_and_slot(
4173 pci_domain_nr(pdev->bus),
4174 bridge->subordinate->number, 0);
3da4af0a 4175 if (!dma_pdev)
abdfdde2
AW		 4176 dma_pdev = pci_dev_get(bridge);
4177 } else
4178 dma_pdev = pci_dev_get(pdev);
4179
a4ff1fc2 4180 /* Account for quirked devices */
783f157b
AW		 4181 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4182
a4ff1fc2
AW		 4183 /*
4184 * If it's a multifunction device that does not support our
4185 * required ACS flags, add to the same group as function 0.
4186 */
783f157b
AW		 4187 if (dma_pdev->multifunction &&
4188 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS))
4189 swap_pci_ref(&dma_pdev,
4190 pci_get_slot(dma_pdev->bus,
4191 PCI_DEVFN(PCI_SLOT(dma_pdev->devfn),
4192 0)));
4193
a4ff1fc2
AW		 4194 /*
4195 * Devices on the root bus go through the iommu. If that's not us,
4196 * find the next upstream device and test ACS up to the root bus.
4197 * Finding the next device may require skipping virtual buses.
4198 */
783f157b 4199 while (!pci_is_root_bus(dma_pdev->bus)) {
a4ff1fc2
AW		 4200 struct pci_bus *bus = dma_pdev->bus;
4201
4202 while (!bus->self) {
4203 if (!pci_is_root_bus(bus))
4204 bus = bus->parent;
4205 else
4206 goto root_bus;
4207 }
4208
4209 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
783f157b
AW		 4210 break;
4211
a4ff1fc2 4212 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
783f157b
AW		 4213 }
4214
a4ff1fc2 4215root_bus:
abdfdde2
AW		 4216 group = iommu_group_get(&dma_pdev->dev);
4217 pci_dev_put(dma_pdev);
4218 if (!group) {
4219 group = iommu_group_alloc();
4220 if (IS_ERR(group))
4221 return PTR_ERR(group);
70ae6f0d
AW		 4222 }
4223
abdfdde2 4224 ret = iommu_group_add_device(group, dev);
bcb71abe 4225
abdfdde2
AW		 4226 iommu_group_put(group);
4227 return ret;
4228}
70ae6f0d 4229
abdfdde2
AW		 4230static void intel_iommu_remove_device(struct device *dev)
4231{
4232 iommu_group_remove_device(dev);
70ae6f0d
AW		 4233}
4234
a8bcbb0d
JR		 4235static struct iommu_ops intel_iommu_ops = {
4236 .domain_init = intel_iommu_domain_init,
4237 .domain_destroy = intel_iommu_domain_destroy,
4238 .attach_dev = intel_iommu_attach_device,
4239 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4240 .map = intel_iommu_map,
4241 .unmap = intel_iommu_unmap,
a8bcbb0d 4242 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4243 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4244 .add_device = intel_iommu_add_device,
4245 .remove_device = intel_iommu_remove_device,
6d1c56a9 4246 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4247};
9af88143 4248
9452618e
DV		 4249static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4250{
4251 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4252 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4253 dmar_map_gfx = 0;
4254}
4255
4256DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4257DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4258DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4259DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4260DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4261DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4262DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4263
d34d6517 4264static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4265{
4266 /*
4267 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4268 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4269 */
4270 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4271 rwbf_quirk = 1;
4272}
4273
4274DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4275DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4276DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4277DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4278DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4279DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4280DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4281
eecfd57f
AJ		 4282#define GGC 0x52
4283#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4284#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4285#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4286#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4287#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4288#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4289#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4290#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4291
d34d6517 4292static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4293{
4294 unsigned short ggc;
4295
eecfd57f 4296 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4297 return;
4298
eecfd57f 4299 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4300 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4301 dmar_map_gfx = 0;
6fbcfb3e
DW		 4302 } else if (dmar_map_gfx) {
4303 /* we have to ensure the gfx device is idle before we flush */
4304 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4305 intel_iommu_strict = 1;
4306 }
9eecabcb
DW		 4307}
4308DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4309DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4310DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4311DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4312
e0fc7e0b
DW		 4313/* On Tylersburg chipsets, some BIOSes have been known to enable the
4314 ISOCH DMAR unit for the Azalia sound device, but not give it any
4315 TLB entries, which causes it to deadlock. Check for that. We do
4316 this in a function called from init_dmars(), instead of in a PCI
4317 quirk, because we don't want to print the obnoxious "BIOS broken"
4318 message if VT-d is actually disabled.
4319*/
4320static void __init check_tylersburg_isoch(void)
4321{
4322 struct pci_dev *pdev;
4323 uint32_t vtisochctrl;
4324
4325 /* If there's no Azalia in the system anyway, forget it. */
4326 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4327 if (!pdev)
4328 return;
4329 pci_dev_put(pdev);
4330
4331 /* System Management Registers. Might be hidden, in which case
4332 we can't do the sanity check. But that's OK, because the
4333 known-broken BIOSes _don't_ actually hide it, so far. */
4334 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4335 if (!pdev)
4336 return;
4337
4338 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4339 pci_dev_put(pdev);
4340 return;
4341 }
4342
4343 pci_dev_put(pdev);
4344
4345 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4346 if (vtisochctrl & 1)
4347 return;
4348
4349 /* Drop all bits other than the number of TLB entries */
4350 vtisochctrl &= 0x1c;
4351
4352 /* If we have the recommended number of TLB entries (16), fine. */
4353 if (vtisochctrl == 0x10)
4354 return;
4355
4356 /* Zero TLB entries? You get to ride the short bus to school. */
4357 if (!vtisochctrl) {
4358 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4359 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4360 dmi_get_system_info(DMI_BIOS_VENDOR),
4361 dmi_get_system_info(DMI_BIOS_VERSION),
4362 dmi_get_system_info(DMI_PRODUCT_VERSION));
4363 iommu_identity_mapping |= IDENTMAP_AZALIA;
4364 return;
4365 }
4366
4367 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4368 vtisochctrl);
4369}
kernel source for telekom puls (alcatel/mediatek)