Intel IOMMU Suspend/Resume Support - Interrupt Remapping

This patch enables suspend/resume for interrupt remapping. During suspend,
interrupt remapping is disabled. When resume, interrupt remapping is enabled
again.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Acked-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>

diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index df8a300dfe6c380b5fdcf60e379ef9fb855d67ae..f9f0866ed6f8dcb7a56219c37b076f6ed69fe786 100644 (file)
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -108,6 +108,10 @@ extern void native_apic_icr_write(u32 low, u32 id);
 extern u64 native_apic_icr_read(void);
 
 #ifdef CONFIG_X86_X2APIC
+
+#define EIM_8BIT_APIC_ID       0
+#define EIM_32BIT_APIC_ID      1
+
 /*
  * Make previous memory operations globally visible before
  * sending the IPI through x2apic wrmsr. We need a serializing instruction or

diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 373cc2bbcad2cdfb17ef4e0448921da892691d0e..9d826e436010005254bb04e13bac1ee45d802b75 100644 (file)
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -162,10 +162,13 @@ extern int (*ioapic_renumber_irq)(int ioapic, int irq);
 extern void ioapic_init_mappings(void);
 
 #ifdef CONFIG_X86_64
-extern int save_IO_APIC_setup(void);
-extern void mask_IO_APIC_setup(void);
-extern void restore_IO_APIC_setup(void);
-extern void reinit_intr_remapped_IO_APIC(int);
+extern struct IO_APIC_route_entry **alloc_ioapic_entries(void);
+extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries);
+extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void reinit_intr_remapped_IO_APIC(int intr_remapping,
+       struct IO_APIC_route_entry **ioapic_entries);
 #endif
 
 extern void probe_nr_irqs_gsi(void);

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 85eb8e100818d2dfef0bae935684e6fe0253c888..098ec84b8c0054d1f0fcff0e02aa357a02b33dd3 100644 (file)
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1304,6 +1304,7 @@ void __init enable_IR_x2apic(void)
 #ifdef CONFIG_INTR_REMAP
        int ret;
        unsigned long flags;
+       struct IO_APIC_route_entry **ioapic_entries = NULL;
 
        if (!cpu_has_x2apic)
                return;
@@ -1334,17 +1335,23 @@ void __init enable_IR_x2apic(void)
                return;
        }
 
-       ret = save_IO_APIC_setup();
+       ioapic_entries = alloc_ioapic_entries();
+       if (!ioapic_entries) {
+               pr_info("Allocate ioapic_entries failed: %d\n", ret);
+               goto end;
+       }
+
+       ret = save_IO_APIC_setup(ioapic_entries);
        if (ret) {
                pr_info("Saving IO-APIC state failed: %d\n", ret);
                goto end;
        }
 
        local_irq_save(flags);
-       mask_IO_APIC_setup();
+       mask_IO_APIC_setup(ioapic_entries);
        mask_8259A();
 
-       ret = enable_intr_remapping(1);
+       ret = enable_intr_remapping(EIM_32BIT_APIC_ID);
 
        if (ret && x2apic_preenabled) {
                local_irq_restore(flags);
@@ -1364,9 +1371,9 @@ end_restore:
                /*
                 * IR enabling failed
                 */
-               restore_IO_APIC_setup();
+               restore_IO_APIC_setup(ioapic_entries);
        else
-               reinit_intr_remapped_IO_APIC(x2apic_preenabled);
+               reinit_intr_remapped_IO_APIC(x2apic_preenabled, ioapic_entries);
 
        unmask_8259A();
        local_irq_restore(flags);
@@ -1379,6 +1386,8 @@ end:
                        pr_info("Enabled Interrupt-remapping\n");
        } else
                pr_err("Failed to enable Interrupt-remapping and x2apic\n");
+       if (ioapic_entries)
+               free_ioapic_entries(ioapic_entries);
 #else
        if (!cpu_has_x2apic)
                return;
@@ -1954,6 +1963,10 @@ static int lapic_suspend(struct sys_device *dev, pm_message_t state)
 
        local_irq_save(flags);
        disable_local_APIC();
+#ifdef CONFIG_INTR_REMAP
+       if (intr_remapping_enabled)
+               disable_intr_remapping();
+#endif
        local_irq_restore(flags);
        return 0;
 }
@@ -1964,15 +1977,41 @@ static int lapic_resume(struct sys_device *dev)
        unsigned long flags;
        int maxlvt;
 
+#ifdef CONFIG_INTR_REMAP
+       int ret;
+       struct IO_APIC_route_entry **ioapic_entries = NULL;
+
        if (!apic_pm_state.active)
                return 0;
 
-       maxlvt = lapic_get_maxlvt();
-
        local_irq_save(flags);
+       if (x2apic) {
+               ioapic_entries = alloc_ioapic_entries();
+               if (!ioapic_entries) {
+                       WARN(1, "Alloc ioapic_entries in lapic resume failed.");
+                       return -ENOMEM;
+               }
+
+               ret = save_IO_APIC_setup(ioapic_entries);
+               if (ret) {
+                       WARN(1, "Saving IO-APIC state failed: %d\n", ret);
+                       free_ioapic_entries(ioapic_entries);
+                       return ret;
+               }
+
+               mask_IO_APIC_setup(ioapic_entries);
+               mask_8259A();
+               enable_x2apic();
+       }
+#else
+       if (!apic_pm_state.active)
+               return 0;
 
+       local_irq_save(flags);
        if (x2apic)
                enable_x2apic();
+#endif
+
        else {
                /*
                 * Make sure the APICBASE points to the right address
@@ -1986,6 +2025,7 @@ static int lapic_resume(struct sys_device *dev)
                wrmsr(MSR_IA32_APICBASE, l, h);
        }
 
+       maxlvt = lapic_get_maxlvt();
        apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
        apic_write(APIC_ID, apic_pm_state.apic_id);
        apic_write(APIC_DFR, apic_pm_state.apic_dfr);
@@ -2009,8 +2049,20 @@ static int lapic_resume(struct sys_device *dev)
        apic_write(APIC_ESR, 0);
        apic_read(APIC_ESR);
 
+#ifdef CONFIG_INTR_REMAP
+       if (intr_remapping_enabled)
+               reenable_intr_remapping(EIM_32BIT_APIC_ID);
+
+       if (x2apic) {
+               unmask_8259A();
+               restore_IO_APIC_setup(ioapic_entries);
+               free_ioapic_entries(ioapic_entries);
+       }
+#endif
+
        local_irq_restore(flags);
 
+
        return 0;
 }
 
@@ -2048,7 +2100,9 @@ static int __init init_lapic_sysfs(void)
                error = sysdev_register(&device_lapic);
        return error;
 }
-device_initcall(init_lapic_sysfs);
+
+/* local apic needs to resume before other devices access its registers. */
+core_initcall(init_lapic_sysfs);
 
 #else  /* CONFIG_PM */
 

diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 1bb5c6cee3ebb140e30e421c05615bf94b001936..0ad3fe77641a7b5c6b797d64e4dc4c6e359f8e13 100644 (file)
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -851,63 +851,74 @@ __setup("pirq=", ioapic_pirq_setup);
 #endif /* CONFIG_X86_32 */
 
 #ifdef CONFIG_INTR_REMAP
-/* I/O APIC RTE contents at the OS boot up */
-static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];
+struct IO_APIC_route_entry **alloc_ioapic_entries(void)
+{
+       int apic;
+       struct IO_APIC_route_entry **ioapic_entries;
+
+       ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
+                               GFP_ATOMIC);
+       if (!ioapic_entries)
+               return 0;
+
+       for (apic = 0; apic < nr_ioapics; apic++) {
+               ioapic_entries[apic] =
+                       kzalloc(sizeof(struct IO_APIC_route_entry) *
+                               nr_ioapic_registers[apic], GFP_ATOMIC);
+               if (!ioapic_entries[apic])
+                       goto nomem;
+       }
+
+       return ioapic_entries;
+
+nomem:
+       while (--apic >= 0)
+               kfree(ioapic_entries[apic]);
+       kfree(ioapic_entries);
+
+       return 0;
+}
 
 /*
  * Saves all the IO-APIC RTE's
  */
-int save_IO_APIC_setup(void)
+int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
 {
-       union IO_APIC_reg_01 reg_01;
-       unsigned long flags;
        int apic, pin;
 
-       /*
-        * The number of IO-APIC IRQ registers (== #pins):
-        */
-       for (apic = 0; apic < nr_ioapics; apic++) {
-               spin_lock_irqsave(&ioapic_lock, flags);
-               reg_01.raw = io_apic_read(apic, 1);
-               spin_unlock_irqrestore(&ioapic_lock, flags);
-               nr_ioapic_registers[apic] = reg_01.bits.entries+1;
-       }
+       if (!ioapic_entries)
+               return -ENOMEM;
 
        for (apic = 0; apic < nr_ioapics; apic++) {
-               early_ioapic_entries[apic] =
-                       kzalloc(sizeof(struct IO_APIC_route_entry) *
-                               nr_ioapic_registers[apic], GFP_KERNEL);
-               if (!early_ioapic_entries[apic])
-                       goto nomem;
-       }
+               if (!ioapic_entries[apic])
+                       return -ENOMEM;
 
-       for (apic = 0; apic < nr_ioapics; apic++)
                for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
-                       early_ioapic_entries[apic][pin] =
+                       ioapic_entries[apic][pin] =
                                ioapic_read_entry(apic, pin);
+       }
 
        return 0;
-
-nomem:
-       while (apic >= 0)
-               kfree(early_ioapic_entries[apic--]);
-       memset(early_ioapic_entries, 0,
-               ARRAY_SIZE(early_ioapic_entries));
-
-       return -ENOMEM;
 }
 
-void mask_IO_APIC_setup(void)
+/*
+ * Mask all IO APIC entries.
+ */
+void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
 {
        int apic, pin;
 
+       if (!ioapic_entries)
+               return;
+
        for (apic = 0; apic < nr_ioapics; apic++) {
-               if (!early_ioapic_entries[apic])
+               if (!ioapic_entries[apic])
                        break;
+
                for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
                        struct IO_APIC_route_entry entry;
 
-                       entry = early_ioapic_entries[apic][pin];
+                       entry = ioapic_entries[apic][pin];
                        if (!entry.mask) {
                                entry.mask = 1;
                                ioapic_write_entry(apic, pin, entry);
@@ -916,22 +927,30 @@ void mask_IO_APIC_setup(void)
        }
 }
 
-void restore_IO_APIC_setup(void)
+/*
+ * Restore IO APIC entries which was saved in ioapic_entries.
+ */
+int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
 {
        int apic, pin;
 
+       if (!ioapic_entries)
+               return -ENOMEM;
+
        for (apic = 0; apic < nr_ioapics; apic++) {
-               if (!early_ioapic_entries[apic])
-                       break;
+               if (!ioapic_entries[apic])
+                       return -ENOMEM;
+
                for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
                        ioapic_write_entry(apic, pin,
-                                          early_ioapic_entries[apic][pin]);
-               kfree(early_ioapic_entries[apic]);
-               early_ioapic_entries[apic] = NULL;
+                                       ioapic_entries[apic][pin]);
        }
+       return 0;
 }
 
-void reinit_intr_remapped_IO_APIC(int intr_remapping)
+void reinit_intr_remapped_IO_APIC(int intr_remapping,
+       struct IO_APIC_route_entry **ioapic_entries)
+
 {
        /*
         * for now plain restore of previous settings.
@@ -940,7 +959,17 @@ void reinit_intr_remapped_IO_APIC(int intr_remapping)
         * table entries. for now, do a plain restore, and wait for
         * the setup_IO_APIC_irqs() to do proper initialization.
         */
-       restore_IO_APIC_setup();
+       restore_IO_APIC_setup(ioapic_entries);
+}
+
+void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
+{
+       int apic;
+
+       for (apic = 0; apic < nr_ioapics; apic++)
+               kfree(ioapic_entries[apic]);
+
+       kfree(ioapic_entries);
 }
 #endif
 

diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c
index b041a409f4a70f84e4a76ed4e0011c758aa31d71..c26633d7e7da75adbe48e63e4ef54bb18a26e9d7 100644 (file)
--- a/drivers/pci/intr_remapping.c
+++ b/drivers/pci/intr_remapping.c
@@ -470,7 +470,7 @@ static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
 /*
  * Disable Interrupt Remapping.
  */
-static void disable_intr_remapping(struct intel_iommu *iommu)
+static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
 {
        unsigned long flags;
        u32 sts;
@@ -478,6 +478,12 @@ static void disable_intr_remapping(struct intel_iommu *iommu)
        if (!ecap_ir_support(iommu->ecap))
                return;
 
+       /*
+        * global invalidation of interrupt entry cache before disabling
+        * interrupt-remapping.
+        */
+       qi_global_iec(iommu);
+
        spin_lock_irqsave(&iommu->register_lock, flags);
 
        sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
@@ -511,7 +517,7 @@ int __init enable_intr_remapping(int eim)
                 * Disable intr remapping and queued invalidation, if already
                 * enabled prior to OS handover.
                 */
-               disable_intr_remapping(iommu);
+               iommu_disable_intr_remapping(iommu);
 
                dmar_disable_qi(iommu);
        }
@@ -639,3 +645,54 @@ int __init parse_ioapics_under_ir(void)
 
        return ir_supported;
 }
+
+void disable_intr_remapping(void)
+{
+       struct dmar_drhd_unit *drhd;
+       struct intel_iommu *iommu = NULL;
+
+       /*
+        * Disable Interrupt-remapping for all the DRHD's now.
+        */
+       for_each_iommu(iommu, drhd) {
+               if (!ecap_ir_support(iommu->ecap))
+                       continue;
+
+               iommu_disable_intr_remapping(iommu);
+       }
+}
+
+int reenable_intr_remapping(int eim)
+{
+       struct dmar_drhd_unit *drhd;
+       int setup = 0;
+       struct intel_iommu *iommu = NULL;
+
+       for_each_iommu(iommu, drhd)
+               if (iommu->qi)
+                       dmar_reenable_qi(iommu);
+
+       /*
+        * Setup Interrupt-remapping for all the DRHD's now.
+        */
+       for_each_iommu(iommu, drhd) {
+               if (!ecap_ir_support(iommu->ecap))
+                       continue;
+
+               /* Set up interrupt remapping for iommu.*/
+               iommu_set_intr_remapping(iommu, eim);
+               setup = 1;
+       }
+
+       if (!setup)
+               goto error;
+
+       return 0;
+
+error:
+       /*
+        * handle error condition gracefully here!
+        */
+       return -1;
+}
+

diff --git a/include/linux/dmar.h b/include/linux/dmar.h
index 0b4aa80699857dbf70e855a3a5a95759b8dfdc96..4a0ce6f27e1b78ceac0357557fc84614dfef82d4 100644 (file)
--- a/include/linux/dmar.h
+++ b/include/linux/dmar.h
@@ -108,6 +108,8 @@ struct irte {
 #ifdef CONFIG_INTR_REMAP
 extern int intr_remapping_enabled;
 extern int enable_intr_remapping(int);
+extern void disable_intr_remapping(void);
+extern int reenable_intr_remapping(int);
 
 extern int get_irte(int irq, struct irte *entry);
 extern int modify_irte(int irq, struct irte *irte_modified);