2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Copyright IBM Corporation, 2008
19 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
21 * Author: Allen M. Kay <allen.m.kay@intel.com>
22 * Author: Weidong Han <weidong.han@intel.com>
23 * Author: Ben-Ami Yassour <benami@il.ibm.com>
26 #include <linux/list.h>
27 #include <linux/kvm_host.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/stat.h>
31 #include <linux/dmar.h>
32 #include <linux/iommu.h>
33 #include <linux/intel-iommu.h>
35 static bool allow_unsafe_assigned_interrupts
;
36 module_param_named(allow_unsafe_assigned_interrupts
,
37 allow_unsafe_assigned_interrupts
, bool, S_IRUGO
| S_IWUSR
);
38 MODULE_PARM_DESC(allow_unsafe_assigned_interrupts
,
39 "Enable device assignment on platforms without interrupt remapping support.");
41 static int kvm_iommu_unmap_memslots(struct kvm
*kvm
);
42 static void kvm_iommu_put_pages(struct kvm
*kvm
,
43 gfn_t base_gfn
, unsigned long npages
);
45 static pfn_t
kvm_pin_pages(struct kvm_memory_slot
*slot
, gfn_t gfn
,
51 pfn
= gfn_to_pfn_memslot(slot
, gfn
);
52 end_gfn
= gfn
+ (size
>> PAGE_SHIFT
);
55 if (is_error_pfn(pfn
))
59 gfn_to_pfn_memslot(slot
, gfn
++);
64 int kvm_iommu_map_pages(struct kvm
*kvm
, struct kvm_memory_slot
*slot
)
69 struct iommu_domain
*domain
= kvm
->arch
.iommu_domain
;
72 /* check if iommu exists and in use */
77 end_gfn
= gfn
+ slot
->npages
;
79 flags
= IOMMU_READ
| IOMMU_WRITE
;
80 if (kvm
->arch
.iommu_flags
& KVM_IOMMU_CACHE_COHERENCY
)
84 while (gfn
< end_gfn
) {
85 unsigned long page_size
;
87 /* Check if already mapped */
88 if (iommu_iova_to_phys(domain
, gfn_to_gpa(gfn
))) {
93 /* Get the page size we could use to map */
94 page_size
= kvm_host_page_size(kvm
, gfn
);
96 /* Make sure the page_size does not exceed the memslot */
97 while ((gfn
+ (page_size
>> PAGE_SHIFT
)) > end_gfn
)
100 /* Make sure gfn is aligned to the page size we want to map */
101 while ((gfn
<< PAGE_SHIFT
) & (page_size
- 1))
105 * Pin all pages we are about to map in memory. This is
106 * important because we unmap and unpin in 4kb steps later.
108 pfn
= kvm_pin_pages(slot
, gfn
, page_size
);
109 if (is_error_pfn(pfn
)) {
110 kvm_release_pfn_clean(pfn
);
115 /* Map into IO address space */
116 r
= iommu_map(domain
, gfn_to_gpa(gfn
), pfn_to_hpa(pfn
),
119 printk(KERN_ERR
"kvm_iommu_map_address:"
120 "iommu failed to map pfn=%llx\n", pfn
);
124 gfn
+= page_size
>> PAGE_SHIFT
;
132 kvm_iommu_put_pages(kvm
, slot
->base_gfn
, gfn
);
136 static int kvm_iommu_map_memslots(struct kvm
*kvm
)
139 struct kvm_memslots
*slots
;
140 struct kvm_memory_slot
*memslot
;
142 idx
= srcu_read_lock(&kvm
->srcu
);
143 slots
= kvm_memslots(kvm
);
145 kvm_for_each_memslot(memslot
, slots
) {
146 r
= kvm_iommu_map_pages(kvm
, memslot
);
150 srcu_read_unlock(&kvm
->srcu
, idx
);
155 int kvm_assign_device(struct kvm
*kvm
,
156 struct kvm_assigned_dev_kernel
*assigned_dev
)
158 struct pci_dev
*pdev
= NULL
;
159 struct iommu_domain
*domain
= kvm
->arch
.iommu_domain
;
162 /* check if iommu exists and in use */
166 pdev
= assigned_dev
->dev
;
170 r
= iommu_attach_device(domain
, &pdev
->dev
);
172 printk(KERN_ERR
"assign device %x:%x:%x.%x failed",
173 pci_domain_nr(pdev
->bus
),
175 PCI_SLOT(pdev
->devfn
),
176 PCI_FUNC(pdev
->devfn
));
180 last_flags
= kvm
->arch
.iommu_flags
;
181 if (iommu_domain_has_cap(kvm
->arch
.iommu_domain
,
182 IOMMU_CAP_CACHE_COHERENCY
))
183 kvm
->arch
.iommu_flags
|= KVM_IOMMU_CACHE_COHERENCY
;
185 /* Check if need to update IOMMU page table for guest memory */
186 if ((last_flags
^ kvm
->arch
.iommu_flags
) ==
187 KVM_IOMMU_CACHE_COHERENCY
) {
188 kvm_iommu_unmap_memslots(kvm
);
189 r
= kvm_iommu_map_memslots(kvm
);
194 pdev
->dev_flags
|= PCI_DEV_FLAGS_ASSIGNED
;
196 printk(KERN_DEBUG
"assign device %x:%x:%x.%x\n",
197 assigned_dev
->host_segnr
,
198 assigned_dev
->host_busnr
,
199 PCI_SLOT(assigned_dev
->host_devfn
),
200 PCI_FUNC(assigned_dev
->host_devfn
));
204 kvm_iommu_unmap_memslots(kvm
);
208 int kvm_deassign_device(struct kvm
*kvm
,
209 struct kvm_assigned_dev_kernel
*assigned_dev
)
211 struct iommu_domain
*domain
= kvm
->arch
.iommu_domain
;
212 struct pci_dev
*pdev
= NULL
;
214 /* check if iommu exists and in use */
218 pdev
= assigned_dev
->dev
;
222 iommu_detach_device(domain
, &pdev
->dev
);
224 pdev
->dev_flags
&= ~PCI_DEV_FLAGS_ASSIGNED
;
226 printk(KERN_DEBUG
"deassign device %x:%x:%x.%x\n",
227 assigned_dev
->host_segnr
,
228 assigned_dev
->host_busnr
,
229 PCI_SLOT(assigned_dev
->host_devfn
),
230 PCI_FUNC(assigned_dev
->host_devfn
));
235 int kvm_iommu_map_guest(struct kvm
*kvm
)
239 if (!iommu_present(&pci_bus_type
)) {
240 printk(KERN_ERR
"%s: iommu not found\n", __func__
);
244 mutex_lock(&kvm
->slots_lock
);
246 kvm
->arch
.iommu_domain
= iommu_domain_alloc(&pci_bus_type
);
247 if (!kvm
->arch
.iommu_domain
) {
252 if (!allow_unsafe_assigned_interrupts
&&
253 !iommu_domain_has_cap(kvm
->arch
.iommu_domain
,
254 IOMMU_CAP_INTR_REMAP
)) {
255 printk(KERN_WARNING
"%s: No interrupt remapping support,"
256 " disallowing device assignment."
257 " Re-enble with \"allow_unsafe_assigned_interrupts=1\""
258 " module option.\n", __func__
);
259 iommu_domain_free(kvm
->arch
.iommu_domain
);
260 kvm
->arch
.iommu_domain
= NULL
;
265 r
= kvm_iommu_map_memslots(kvm
);
267 kvm_iommu_unmap_memslots(kvm
);
270 mutex_unlock(&kvm
->slots_lock
);
274 static void kvm_unpin_pages(struct kvm
*kvm
, pfn_t pfn
, unsigned long npages
)
278 for (i
= 0; i
< npages
; ++i
)
279 kvm_release_pfn_clean(pfn
+ i
);
282 static void kvm_iommu_put_pages(struct kvm
*kvm
,
283 gfn_t base_gfn
, unsigned long npages
)
285 struct iommu_domain
*domain
;
290 domain
= kvm
->arch
.iommu_domain
;
291 end_gfn
= base_gfn
+ npages
;
294 /* check if iommu exists and in use */
298 while (gfn
< end_gfn
) {
299 unsigned long unmap_pages
;
302 /* Get physical address */
303 phys
= iommu_iova_to_phys(domain
, gfn_to_gpa(gfn
));
310 pfn
= phys
>> PAGE_SHIFT
;
312 /* Unmap address from IO address space */
313 size
= iommu_unmap(domain
, gfn_to_gpa(gfn
), PAGE_SIZE
);
314 unmap_pages
= 1ULL << get_order(size
);
316 /* Unpin all pages we just unmapped to not leak any memory */
317 kvm_unpin_pages(kvm
, pfn
, unmap_pages
);
323 void kvm_iommu_unmap_pages(struct kvm
*kvm
, struct kvm_memory_slot
*slot
)
325 kvm_iommu_put_pages(kvm
, slot
->base_gfn
, slot
->npages
);
328 static int kvm_iommu_unmap_memslots(struct kvm
*kvm
)
331 struct kvm_memslots
*slots
;
332 struct kvm_memory_slot
*memslot
;
334 idx
= srcu_read_lock(&kvm
->srcu
);
335 slots
= kvm_memslots(kvm
);
337 kvm_for_each_memslot(memslot
, slots
)
338 kvm_iommu_unmap_pages(kvm
, memslot
);
340 srcu_read_unlock(&kvm
->srcu
, idx
);
345 int kvm_iommu_unmap_guest(struct kvm
*kvm
)
347 struct iommu_domain
*domain
= kvm
->arch
.iommu_domain
;
349 /* check if iommu exists and in use */
353 mutex_lock(&kvm
->slots_lock
);
354 kvm_iommu_unmap_memslots(kvm
);
355 kvm
->arch
.iommu_domain
= NULL
;
356 mutex_unlock(&kvm
->slots_lock
);
358 iommu_domain_free(domain
);