unsigned long address, int map_size);
static void update_domain(struct protection_domain *domain);
+/****************************************************************************
+ *
+ * Helper functions
+ *
+ ****************************************************************************/
+
+static inline u16 get_device_id(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+
+ return calc_devid(pdev->bus->number, pdev->devfn);
+}
+
#ifdef CONFIG_AMD_IOMMU_STATS
/*
return domain->flags & PD_DMA_OPS_MASK;
}
-/*
- * Find out the protection domain structure for a given PCI device. This
- * will give us the pointer to the page table root for example.
- */
-static struct protection_domain *domain_for_device(u16 devid)
-{
- struct protection_domain *dom;
- unsigned long flags;
-
- read_lock_irqsave(&amd_iommu_devtable_lock, flags);
- dom = amd_iommu_pd_table[devid];
- read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
-
- return dom;
-}
-
static void set_dte_entry(u16 devid, struct protection_domain *domain)
{
+ struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
u64 pte_root = virt_to_phys(domain->pt_root);
+ BUG_ON(amd_iommu_pd_table[devid] != NULL);
+
pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;
pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
amd_iommu_pd_table[devid] = domain;
+
+ /* Do reference counting */
+ domain->dev_iommu[iommu->index] += 1;
+ domain->dev_cnt += 1;
+
+ /* Flush the changes DTE entry */
+ iommu_queue_inv_dev_entry(iommu, devid);
+}
+
+static void clear_dte_entry(u16 devid)
+{
+ struct protection_domain *domain = amd_iommu_pd_table[devid];
+ struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+
+ BUG_ON(domain == NULL);
+
+ /* remove domain from the lookup table */
+ amd_iommu_pd_table[devid] = NULL;
+
+ /* remove entry from the device table seen by the hardware */
+ amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
+ amd_iommu_dev_table[devid].data[1] = 0;
+ amd_iommu_dev_table[devid].data[2] = 0;
+
+ amd_iommu_apply_erratum_63(devid);
+
+ /* decrease reference counters */
+ domain->dev_iommu[iommu->index] -= 1;
+ domain->dev_cnt -= 1;
+
+ iommu_queue_inv_dev_entry(iommu, devid);
}
/*
* If a device is not yet associated with a domain, this function does
* assigns it visible for the hardware
*/
-static void __attach_device(struct amd_iommu *iommu,
- struct protection_domain *domain,
- u16 devid)
+static int __attach_device(struct device *dev,
+ struct protection_domain *domain)
{
+ u16 devid = get_device_id(dev);
+ u16 alias = amd_iommu_alias_table[devid];
+
/* lock domain */
spin_lock(&domain->lock);
- /* update DTE entry */
- set_dte_entry(devid, domain);
+ /* Some sanity checks */
+ if (amd_iommu_pd_table[alias] != NULL &&
+ amd_iommu_pd_table[alias] != domain)
+ return -EBUSY;
- /* Do reference counting */
- domain->dev_iommu[iommu->index] += 1;
- domain->dev_cnt += 1;
+ if (amd_iommu_pd_table[devid] != NULL &&
+ amd_iommu_pd_table[devid] != domain)
+ return -EBUSY;
+
+ /* Do real assignment */
+ if (alias != devid &&
+ amd_iommu_pd_table[alias] == NULL)
+ set_dte_entry(alias, domain);
+
+ if (amd_iommu_pd_table[devid] == NULL)
+ set_dte_entry(devid, domain);
/* ready */
spin_unlock(&domain->lock);
+
+ return 0;
}
/*
* If a device is not yet associated with a domain, this function does
* assigns it visible for the hardware
*/
-static void attach_device(struct amd_iommu *iommu,
- struct protection_domain *domain,
- u16 devid)
+static int attach_device(struct device *dev,
+ struct protection_domain *domain)
{
unsigned long flags;
+ int ret;
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
- __attach_device(iommu, domain, devid);
+ ret = __attach_device(dev, domain);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
/*
* left the caches in the IOMMU dirty. So we have to flush
* here to evict all dirty stuff.
*/
- iommu_queue_inv_dev_entry(iommu, devid);
iommu_flush_tlb_pde(domain);
+
+ return ret;
}
/*
* Removes a device from a protection domain (unlocked)
*/
-static void __detach_device(struct protection_domain *domain, u16 devid)
+static void __detach_device(struct device *dev)
{
+ u16 devid = get_device_id(dev);
struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
BUG_ON(!iommu);
- /* lock domain */
- spin_lock(&domain->lock);
-
- /* remove domain from the lookup table */
- amd_iommu_pd_table[devid] = NULL;
-
- /* remove entry from the device table seen by the hardware */
- amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
- amd_iommu_dev_table[devid].data[1] = 0;
- amd_iommu_dev_table[devid].data[2] = 0;
-
- amd_iommu_apply_erratum_63(devid);
-
- /* decrease reference counters */
- domain->dev_iommu[iommu->index] -= 1;
- domain->dev_cnt -= 1;
-
- /* ready */
- spin_unlock(&domain->lock);
+ clear_dte_entry(devid);
/*
* If we run in passthrough mode the device must be assigned to the
* passthrough domain if it is detached from any other domain
*/
- if (iommu_pass_through) {
- struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
- __attach_device(iommu, pt_domain, devid);
- }
+ if (iommu_pass_through)
+ __attach_device(dev, pt_domain);
}
/*
* Removes a device from a protection domain (with devtable_lock held)
*/
-static void detach_device(struct protection_domain *domain, u16 devid)
+static void detach_device(struct device *dev)
{
unsigned long flags;
/* lock device table */
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
- __detach_device(domain, devid);
+ __detach_device(dev);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}
+/*
+ * Find out the protection domain structure for a given PCI device. This
+ * will give us the pointer to the page table root for example.
+ */
+static struct protection_domain *domain_for_device(struct device *dev)
+{
+ struct protection_domain *dom;
+ unsigned long flags;
+ u16 devid, alias;
+
+ devid = get_device_id(dev);
+ alias = amd_iommu_alias_table[devid];
+
+ read_lock_irqsave(&amd_iommu_devtable_lock, flags);
+ dom = amd_iommu_pd_table[devid];
+ if (dom == NULL &&
+ amd_iommu_pd_table[alias] != NULL) {
+ __attach_device(dev, amd_iommu_pd_table[alias]);
+ dom = amd_iommu_pd_table[devid];
+ }
+
+ read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+
+ return dom;
+}
+
static int device_change_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
if (iommu == NULL)
goto out;
- domain = domain_for_device(devid);
+ domain = domain_for_device(dev);
if (domain && !dma_ops_domain(domain))
WARN_ONCE(1, "AMD IOMMU WARNING: device %s already bound "
goto out;
if (iommu_pass_through)
break;
- detach_device(domain, devid);
+ detach_device(dev);
break;
case BUS_NOTIFY_ADD_DEVICE:
/* allocate a protection domain if a device is added */
{
struct dma_ops_domain *dma_dom;
struct amd_iommu *iommu;
- struct pci_dev *pcidev;
- u16 _bdf;
if (!check_device(dev))
return false;
- pcidev = to_pci_dev(dev);
- _bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
- *bdf = amd_iommu_alias_table[_bdf];
+ *bdf = get_device_id(dev);
+ *domain = domain_for_device(dev);
iommu = amd_iommu_rlookup_table[*bdf];
- *domain = domain_for_device(*bdf);
- if (*domain == NULL) {
- dma_dom = find_protection_domain(*bdf);
- if (!dma_dom)
- dma_dom = iommu->default_dom;
- *domain = &dma_dom->domain;
- attach_device(iommu, *domain, *bdf);
- DUMP_printk("Using protection domain %d for device %s\n",
- (*domain)->id, dev_name(dev));
- }
+ if (*domain != NULL)
+ return true;
- if (domain_for_device(_bdf) == NULL)
- attach_device(iommu, *domain, _bdf);
+ /* Device not bount yet - bind it */
+ dma_dom = find_protection_domain(*bdf);
+ if (!dma_dom)
+ dma_dom = iommu->default_dom;
+ *domain = &dma_dom->domain;
+ attach_device(dev, *domain);
+ DUMP_printk("Using protection domain %d for device %s\n",
+ (*domain)->id, dev_name(dev));
return true;
}
if (devid > amd_iommu_last_bdf)
continue;
devid = amd_iommu_alias_table[devid];
- if (domain_for_device(devid))
+ if (domain_for_device(&dev->dev))
continue;
iommu = amd_iommu_rlookup_table[devid];
if (!iommu)
init_unity_mappings_for_device(dma_dom, devid);
dma_dom->target_dev = devid;
- attach_device(iommu, &dma_dom->domain, devid);
- if (__devid != devid)
- attach_device(iommu, &dma_dom->domain, __devid);
+ attach_device(&dev->dev, &dma_dom->domain);
list_add_tail(&dma_dom->list, &iommu_pd_list);
}
for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
if (amd_iommu_pd_table[devid] == domain)
- __detach_device(domain, devid);
+ clear_dte_entry(devid);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}
static void amd_iommu_detach_device(struct iommu_domain *dom,
struct device *dev)
{
- struct protection_domain *domain = dom->priv;
struct amd_iommu *iommu;
struct pci_dev *pdev;
u16 devid;
devid = calc_devid(pdev->bus->number, pdev->devfn);
if (devid > 0)
- detach_device(domain, devid);
+ detach_device(dev);
iommu = amd_iommu_rlookup_table[devid];
if (!iommu)
struct protection_domain *old_domain;
struct amd_iommu *iommu;
struct pci_dev *pdev;
+ int ret;
u16 devid;
if (dev->bus != &pci_bus_type)
if (!iommu)
return -EINVAL;
- old_domain = domain_for_device(devid);
+ old_domain = amd_iommu_pd_table[devid];
if (old_domain)
- detach_device(old_domain, devid);
+ detach_device(dev);
- attach_device(iommu, domain, devid);
+ ret = attach_device(dev, domain);
iommu_completion_wait(iommu);
- return 0;
+ return ret;
}
static int amd_iommu_map_range(struct iommu_domain *dom,
int __init amd_iommu_init_passthrough(void)
{
+ struct amd_iommu *iommu;
struct pci_dev *dev = NULL;
- u16 devid, devid2;
+ u16 devid;
/* allocate passthroug domain */
pt_domain = protection_domain_alloc();
pt_domain->mode |= PAGE_MODE_NONE;
while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
- struct amd_iommu *iommu;
devid = calc_devid(dev->bus->number, dev->devfn);
if (devid > amd_iommu_last_bdf)
continue;
- devid2 = amd_iommu_alias_table[devid];
-
- iommu = amd_iommu_rlookup_table[devid2];
+ iommu = amd_iommu_rlookup_table[devid];
if (!iommu)
continue;
- __attach_device(iommu, pt_domain, devid);
- __attach_device(iommu, pt_domain, devid2);
+ attach_device(&dev->dev, pt_domain);
}
pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
projects / GitHub / exynos8895 / android_kernel_samsung_universal8895.git / commitdiff