1 #include <linux/seq_file.h>
2 #include <linux/cpumask.h>
3 #include <linux/kernel.h>
4 #include <linux/string.h>
5 #include <linux/errno.h>
10 #include <asm/hw_irq.h>
11 #include <asm/irq_remapping.h>
12 #include <asm/processor.h>
13 #include <asm/x86_init.h>
16 #include "irq_remapping.h"
18 int irq_remapping_enabled
;
20 int disable_irq_remap
;
21 int disable_sourceid_checking
;
24 static struct irq_remap_ops
*remap_ops
;
26 static int msi_alloc_remapped_irq(struct pci_dev
*pdev
, int irq
, int nvec
);
27 static int msi_setup_remapped_irq(struct pci_dev
*pdev
, unsigned int irq
,
28 int index
, int sub_handle
);
29 static int set_remapped_irq_affinity(struct irq_data
*data
,
30 const struct cpumask
*mask
,
33 static bool irq_remapped(struct irq_cfg
*cfg
)
35 return (cfg
->remapped
== 1);
38 static void irq_remapping_disable_io_apic(void)
41 * With interrupt-remapping, for now we will use virtual wire A
42 * mode, as virtual wire B is little complex (need to configure
43 * both IOAPIC RTE as well as interrupt-remapping table entry).
44 * As this gets called during crash dump, keep this simple for
47 if (cpu_has_apic
|| apic_from_smp_config())
48 disconnect_bsp_APIC(0);
51 static int do_setup_msi_irqs(struct pci_dev
*dev
, int nvec
)
53 int node
, ret
, sub_handle
, index
= 0;
55 struct msi_desc
*msidesc
;
57 nvec
= __roundup_pow_of_two(nvec
);
59 WARN_ON(!list_is_singular(&dev
->msi_list
));
60 msidesc
= list_entry(dev
->msi_list
.next
, struct msi_desc
, list
);
61 WARN_ON(msidesc
->irq
);
62 WARN_ON(msidesc
->msi_attrib
.multiple
);
64 node
= dev_to_node(&dev
->dev
);
65 irq
= __create_irqs(get_nr_irqs_gsi(), nvec
, node
);
69 msidesc
->msi_attrib
.multiple
= ilog2(nvec
);
70 for (sub_handle
= 0; sub_handle
< nvec
; sub_handle
++) {
72 index
= msi_alloc_remapped_irq(dev
, irq
, nvec
);
78 ret
= msi_setup_remapped_irq(dev
, irq
+ sub_handle
,
83 ret
= setup_msi_irq(dev
, msidesc
, irq
, sub_handle
);
90 destroy_irqs(irq
, nvec
);
93 * Restore altered MSI descriptor fields and prevent just destroyed
94 * IRQs from tearing down again in default_teardown_msi_irqs()
97 msidesc
->msi_attrib
.multiple
= 0;
102 static int do_setup_msix_irqs(struct pci_dev
*dev
, int nvec
)
104 int node
, ret
, sub_handle
, index
= 0;
105 struct msi_desc
*msidesc
;
108 node
= dev_to_node(&dev
->dev
);
109 irq
= get_nr_irqs_gsi();
112 list_for_each_entry(msidesc
, &dev
->msi_list
, list
) {
114 irq
= create_irq_nr(irq
, node
);
119 ret
= index
= msi_alloc_remapped_irq(dev
, irq
, nvec
);
121 ret
= msi_setup_remapped_irq(dev
, irq
, index
, sub_handle
);
126 ret
= setup_msi_irq(dev
, msidesc
, irq
, 0);
141 static int irq_remapping_setup_msi_irqs(struct pci_dev
*dev
,
144 if (type
== PCI_CAP_ID_MSI
)
145 return do_setup_msi_irqs(dev
, nvec
);
147 return do_setup_msix_irqs(dev
, nvec
);
150 void eoi_ioapic_pin_remapped(int apic
, int pin
, int vector
)
153 * Intr-remapping uses pin number as the virtual vector
154 * in the RTE. Actual vector is programmed in
155 * intr-remapping table entry. Hence for the io-apic
156 * EOI we use the pin number.
158 io_apic_eoi(apic
, pin
);
161 static void __init
irq_remapping_modify_x86_ops(void)
163 x86_io_apic_ops
.disable
= irq_remapping_disable_io_apic
;
164 x86_io_apic_ops
.set_affinity
= set_remapped_irq_affinity
;
165 x86_io_apic_ops
.setup_entry
= setup_ioapic_remapped_entry
;
166 x86_io_apic_ops
.eoi_ioapic_pin
= eoi_ioapic_pin_remapped
;
167 x86_msi
.setup_msi_irqs
= irq_remapping_setup_msi_irqs
;
168 x86_msi
.setup_hpet_msi
= setup_hpet_msi_remapped
;
169 x86_msi
.compose_msi_msg
= compose_remapped_msi_msg
;
172 static __init
int setup_nointremap(char *str
)
174 disable_irq_remap
= 1;
177 early_param("nointremap", setup_nointremap
);
179 static __init
int setup_irqremap(char *str
)
185 if (!strncmp(str
, "on", 2))
186 disable_irq_remap
= 0;
187 else if (!strncmp(str
, "off", 3))
188 disable_irq_remap
= 1;
189 else if (!strncmp(str
, "nosid", 5))
190 disable_sourceid_checking
= 1;
191 else if (!strncmp(str
, "no_x2apic_optout", 16))
192 no_x2apic_optout
= 1;
194 str
+= strcspn(str
, ",");
201 early_param("intremap", setup_irqremap
);
203 void __init
setup_irq_remapping_ops(void)
205 remap_ops
= &intel_irq_remap_ops
;
207 #ifdef CONFIG_AMD_IOMMU
208 if (amd_iommu_irq_ops
.prepare() == 0)
209 remap_ops
= &amd_iommu_irq_ops
;
213 int irq_remapping_supported(void)
215 if (disable_irq_remap
)
218 if (!remap_ops
|| !remap_ops
->supported
)
221 return remap_ops
->supported();
224 int __init
irq_remapping_prepare(void)
226 if (!remap_ops
|| !remap_ops
->prepare
)
229 return remap_ops
->prepare();
232 int __init
irq_remapping_enable(void)
236 if (!remap_ops
|| !remap_ops
->enable
)
239 ret
= remap_ops
->enable();
241 if (irq_remapping_enabled
)
242 irq_remapping_modify_x86_ops();
247 void irq_remapping_disable(void)
249 if (!irq_remapping_enabled
||
254 remap_ops
->disable();
257 int irq_remapping_reenable(int mode
)
259 if (!irq_remapping_enabled
||
261 !remap_ops
->reenable
)
264 return remap_ops
->reenable(mode
);
267 int __init
irq_remap_enable_fault_handling(void)
269 if (!irq_remapping_enabled
)
272 if (!remap_ops
|| !remap_ops
->enable_faulting
)
275 return remap_ops
->enable_faulting();
278 int setup_ioapic_remapped_entry(int irq
,
279 struct IO_APIC_route_entry
*entry
,
280 unsigned int destination
, int vector
,
281 struct io_apic_irq_attr
*attr
)
283 if (!remap_ops
|| !remap_ops
->setup_ioapic_entry
)
286 return remap_ops
->setup_ioapic_entry(irq
, entry
, destination
,
290 int set_remapped_irq_affinity(struct irq_data
*data
, const struct cpumask
*mask
,
293 if (!config_enabled(CONFIG_SMP
) || !remap_ops
||
294 !remap_ops
->set_affinity
)
297 return remap_ops
->set_affinity(data
, mask
, force
);
300 void free_remapped_irq(int irq
)
302 struct irq_cfg
*cfg
= irq_get_chip_data(irq
);
304 if (!remap_ops
|| !remap_ops
->free_irq
)
307 if (irq_remapped(cfg
))
308 remap_ops
->free_irq(irq
);
311 void compose_remapped_msi_msg(struct pci_dev
*pdev
,
312 unsigned int irq
, unsigned int dest
,
313 struct msi_msg
*msg
, u8 hpet_id
)
315 struct irq_cfg
*cfg
= irq_get_chip_data(irq
);
317 if (!irq_remapped(cfg
))
318 native_compose_msi_msg(pdev
, irq
, dest
, msg
, hpet_id
);
319 else if (remap_ops
&& remap_ops
->compose_msi_msg
)
320 remap_ops
->compose_msi_msg(pdev
, irq
, dest
, msg
, hpet_id
);
323 static int msi_alloc_remapped_irq(struct pci_dev
*pdev
, int irq
, int nvec
)
325 if (!remap_ops
|| !remap_ops
->msi_alloc_irq
)
328 return remap_ops
->msi_alloc_irq(pdev
, irq
, nvec
);
331 static int msi_setup_remapped_irq(struct pci_dev
*pdev
, unsigned int irq
,
332 int index
, int sub_handle
)
334 if (!remap_ops
|| !remap_ops
->msi_setup_irq
)
337 return remap_ops
->msi_setup_irq(pdev
, irq
, index
, sub_handle
);
340 int setup_hpet_msi_remapped(unsigned int irq
, unsigned int id
)
342 if (!remap_ops
|| !remap_ops
->setup_hpet_msi
)
345 return remap_ops
->setup_hpet_msi(irq
, id
);
348 void panic_if_irq_remap(const char *msg
)
350 if (irq_remapping_enabled
)
354 static void ir_ack_apic_edge(struct irq_data
*data
)
359 static void ir_ack_apic_level(struct irq_data
*data
)
362 eoi_ioapic_irq(data
->irq
, data
->chip_data
);
365 static void ir_print_prefix(struct irq_data
*data
, struct seq_file
*p
)
367 seq_printf(p
, " IR-%s", data
->chip
->name
);
370 void irq_remap_modify_chip_defaults(struct irq_chip
*chip
)
372 chip
->irq_print_chip
= ir_print_prefix
;
373 chip
->irq_ack
= ir_ack_apic_edge
;
374 chip
->irq_eoi
= ir_ack_apic_level
;
375 chip
->irq_set_affinity
= x86_io_apic_ops
.set_affinity
;
378 bool setup_remapped_irq(int irq
, struct irq_cfg
*cfg
, struct irq_chip
*chip
)
380 if (!irq_remapped(cfg
))
382 irq_set_status_flags(irq
, IRQ_MOVE_PCNTXT
);
383 irq_remap_modify_chip_defaults(chip
);