iommu: Add iommu_map_sg() function

Mapping and unmapping are more often than not in the critical path.
map_sg allows IOMMU driver implementations to optimize the process
of mapping buffers into the IOMMU page tables.

Instead of mapping a buffer one page at a time and requiring potentially
expensive TLB operations for each page, this function allows the driver
to map all pages in one go and defer TLB maintenance until after all
pages have been mapped.

Additionally, the mapping operation would be faster in general since
clients does not have to keep calling map API over and over again for
each physically contiguous chunk of memory that needs to be mapped to a
virtually contiguous region.

Signed-off-by: Olav Haugan <ohaugan@codeaurora.org>
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 505a9adac2d51f560617ae36c7a2f53a58f906b6..2d84c9edf3b88e4017c2ba34cb83a2a96d638274 100644 (file)
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3424,6 +3424,7 @@ static const struct iommu_ops amd_iommu_ops = {
        .detach_dev = amd_iommu_detach_device,
        .map = amd_iommu_map,
        .unmap = amd_iommu_unmap,
+       .map_sg = default_iommu_map_sg,
        .iova_to_phys = amd_iommu_iova_to_phys,
        .pgsize_bitmap  = AMD_IOMMU_PGSIZES,
 };

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 60558f794922f9c7c0b5f690b195fd133c1a730a..e393ae01b5d28316bce3fde017635bbe653aef21 100644 (file)
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1652,6 +1652,7 @@ static const struct iommu_ops arm_smmu_ops = {
        .detach_dev     = arm_smmu_detach_dev,
        .map            = arm_smmu_map,
        .unmap          = arm_smmu_unmap,
+       .map_sg         = default_iommu_map_sg,
        .iova_to_phys   = arm_smmu_iova_to_phys,
        .add_device     = arm_smmu_add_device,
        .remove_device  = arm_smmu_remove_device,

diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 74233186f6f7b0311cb2c712a2890f91ada9e458..28372b85d8da5576db75403bc88f9a1437564bfa 100644 (file)
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1178,6 +1178,7 @@ static const struct iommu_ops exynos_iommu_ops = {
        .detach_dev = exynos_iommu_detach_device,
        .map = exynos_iommu_map,
        .unmap = exynos_iommu_unmap,
+       .map_sg = default_iommu_map_sg,
        .iova_to_phys = exynos_iommu_iova_to_phys,
        .add_device = exynos_iommu_add_device,
        .remove_device = exynos_iommu_remove_device,

diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index a27d6cb1a793edb5559e09ee49c993817ff8ebc0..02cd26a17fe071f18591d10f8ece0e2d91686381 100644 (file)
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -4467,6 +4467,7 @@ static const struct iommu_ops intel_iommu_ops = {
        .detach_dev     = intel_iommu_detach_device,
        .map            = intel_iommu_map,
        .unmap          = intel_iommu_unmap,
+       .map_sg         = default_iommu_map_sg,
        .iova_to_phys   = intel_iommu_iova_to_phys,
        .add_device     = intel_iommu_add_device,
        .remove_device  = intel_iommu_remove_device,

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index ed8b04867b1fce059f2fd0ee3c242e161cdb1e52..46727ce9280dc7296feb7d130f362851d5799587 100644 (file)
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -1124,6 +1124,31 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
 }
 EXPORT_SYMBOL_GPL(iommu_unmap);
 
+size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
+                        struct scatterlist *sg, unsigned int nents, int prot)
+{
+       int ret;
+       size_t mapped = 0;
+       unsigned int i;
+       struct scatterlist *s;
+
+       for_each_sg(sg, s, nents, i) {
+               phys_addr_t phys = page_to_phys(sg_page(s));
+               size_t page_len = s->offset + s->length;
+
+               ret = iommu_map(domain, iova + mapped, phys, page_len, prot);
+               if (ret) {
+                       /* undo mappings already done */
+                       iommu_unmap(domain, iova, mapped);
+                       mapped = 0;
+                       break;
+               }
+               mapped += page_len;
+       }
+
+       return mapped;
+}
+EXPORT_SYMBOL_GPL(default_iommu_map_sg);
 
 int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
                               phys_addr_t paddr, u64 size, int prot)

diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 7dab5cbcc775414e5d8e20b48cedf6fed439a7d3..e509c58eee92ac9ccfda3ea13d7f0617cb900af1 100644 (file)
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -1127,6 +1127,7 @@ static const struct iommu_ops ipmmu_ops = {
        .detach_dev = ipmmu_detach_device,
        .map = ipmmu_map,
        .unmap = ipmmu_unmap,
+       .map_sg = default_iommu_map_sg,
        .iova_to_phys = ipmmu_iova_to_phys,
        .add_device = ipmmu_add_device,
        .remove_device = ipmmu_remove_device,

diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 6e3dcc289d59d49c2cec89a4c0abcba19c3b01bd..1c7b78ecf3e307718fe52af9d5fa32b5f2a8ff22 100644 (file)
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -681,6 +681,7 @@ static const struct iommu_ops msm_iommu_ops = {
        .detach_dev = msm_iommu_detach_dev,
        .map = msm_iommu_map,
        .unmap = msm_iommu_unmap,
+       .map_sg = default_iommu_map_sg,
        .iova_to_phys = msm_iommu_iova_to_phys,
        .pgsize_bitmap = MSM_IOMMU_PGSIZES,
 };

diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 36278870e84a7b1d7350772beb2c8e2aee11fa40..18003c0444541bdbee5915eee228e7d8e2c87731 100644 (file)
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -1288,6 +1288,7 @@ static const struct iommu_ops omap_iommu_ops = {
        .detach_dev     = omap_iommu_detach_dev,
        .map            = omap_iommu_map,
        .unmap          = omap_iommu_unmap,
+       .map_sg         = default_iommu_map_sg,
        .iova_to_phys   = omap_iommu_iova_to_phys,
        .add_device     = omap_iommu_add_device,
        .remove_device  = omap_iommu_remove_device,

diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c
index 1333e6fb3405a34513068f1ca3f4be02d99d5120..f1b00774e4de980511cf741895a219059f52d6fa 100644 (file)
--- a/drivers/iommu/shmobile-iommu.c
+++ b/drivers/iommu/shmobile-iommu.c
@@ -361,6 +361,7 @@ static const struct iommu_ops shmobile_iommu_ops = {
        .detach_dev = shmobile_iommu_detach_device,
        .map = shmobile_iommu_map,
        .unmap = shmobile_iommu_unmap,
+       .map_sg = default_iommu_map_sg,
        .iova_to_phys = shmobile_iommu_iova_to_phys,
        .add_device = shmobile_iommu_add_device,
        .pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K,

diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 3afdf43f732ac0df57e0893d37a71f1c2ca6d201..73e845a66925d20f37c8a93be623f01df1242a98 100644 (file)
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -955,6 +955,7 @@ static const struct iommu_ops smmu_iommu_ops = {
        .detach_dev     = smmu_iommu_detach_dev,
        .map            = smmu_iommu_map,
        .unmap          = smmu_iommu_unmap,
+       .map_sg         = default_iommu_map_sg,
        .iova_to_phys   = smmu_iommu_iova_to_phys,
        .pgsize_bitmap  = SMMU_IOMMU_PGSIZES,
 };

diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index e6a7c9ff72f2a717ba643d4a2a7786f93a33497e..b29a5982e1c3381052a170e7c0fa898e61147a2f 100644 (file)
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -22,6 +22,7 @@
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/types.h>
+#include <linux/scatterlist.h>
 #include <trace/events/iommu.h>
 
 #define IOMMU_READ     (1 << 0)
@@ -97,6 +98,8 @@ enum iommu_attr {
  * @detach_dev: detach device from an iommu domain
  * @map: map a physically contiguous memory region to an iommu domain
  * @unmap: unmap a physically contiguous memory region from an iommu domain
+ * @map_sg: map a scatter-gather list of physically contiguous memory chunks
+ * to an iommu domain
  * @iova_to_phys: translate iova to physical address
  * @add_device: add device to iommu grouping
  * @remove_device: remove device from iommu grouping
@@ -114,6 +117,8 @@ struct iommu_ops {
                   phys_addr_t paddr, size_t size, int prot);
        size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
                     size_t size);
+       size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova,
+                        struct scatterlist *sg, unsigned int nents, int prot);
        phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);
        int (*add_device)(struct device *dev);
        void (*remove_device)(struct device *dev);
@@ -156,6 +161,9 @@ extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
                     phys_addr_t paddr, size_t size, int prot);
 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
                       size_t size);
+extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
+                               struct scatterlist *sg,unsigned int nents,
+                               int prot);
 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
 extern void iommu_set_fault_handler(struct iommu_domain *domain,
                        iommu_fault_handler_t handler, void *token);
@@ -241,6 +249,13 @@ static inline int report_iommu_fault(struct iommu_domain *domain,
        return ret;
 }
 
+static inline size_t iommu_map_sg(struct iommu_domain *domain,
+                                 unsigned long iova, struct scatterlist *sg,
+                                 unsigned int nents, int prot)
+{
+       return domain->ops->map_sg(domain, iova, sg, nents, prot);
+}
+
 #else /* CONFIG_IOMMU_API */
 
 struct iommu_ops {};
@@ -293,6 +308,13 @@ static inline int iommu_unmap(struct iommu_domain *domain, unsigned long iova,
        return -ENODEV;
 }
 
+static inline size_t iommu_map_sg(struct iommu_domain *domain,
+                                 unsigned long iova, struct scatterlist *sg,
+                                 unsigned int nents, int prot)
+{
+       return -ENODEV;
+}
+
 static inline int iommu_domain_window_enable(struct iommu_domain *domain,
                                             u32 wnd_nr, phys_addr_t paddr,
                                             u64 size, int prot)