* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
- * Copyright (C) 2004-2008, 2009, 2010 Cavium Networks
+ * Copyright (C) 2004-2008, 2009, 2010, 2011 Cavium Networks
*/
-#include <linux/irq.h>
+
#include <linux/interrupt.h>
+#include <linux/bitops.h>
+#include <linux/percpu.h>
+#include <linux/irq.h>
#include <linux/smp.h>
#include <asm/octeon/octeon.h>
static DEFINE_RAW_SPINLOCK(octeon_irq_ciu0_lock);
static DEFINE_RAW_SPINLOCK(octeon_irq_ciu1_lock);
+static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
+static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
+
+static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];
+
+union octeon_ciu_chip_data {
+ void *p;
+ unsigned long l;
+ struct {
+ unsigned int line:6;
+ unsigned int bit:6;
+ } s;
+};
+
+struct octeon_core_chip_data {
+ struct mutex core_irq_mutex;
+ bool current_en;
+ bool desired_en;
+ u8 bit;
+};
+
+#define MIPS_CORE_IRQ_LINES 8
+
+static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];
+
+static void __init octeon_irq_set_ciu_mapping(int irq, int line, int bit,
+ struct irq_chip *chip,
+ irq_flow_handler_t handler)
+{
+ union octeon_ciu_chip_data cd;
+
+ irq_set_chip_and_handler(irq, chip, handler);
+
+ cd.l = 0;
+ cd.s.line = line;
+ cd.s.bit = bit;
+
+ irq_set_chip_data(irq, cd.p);
+ octeon_irq_ciu_to_irq[line][bit] = irq;
+}
+
static int octeon_coreid_for_cpu(int cpu)
{
#ifdef CONFIG_SMP
#endif
}
-static void octeon_irq_core_ack(unsigned int irq)
+static int octeon_cpu_for_coreid(int coreid)
+{
+#ifdef CONFIG_SMP
+ return cpu_number_map(coreid);
+#else
+ return smp_processor_id();
+#endif
+}
+
+static void octeon_irq_core_ack(struct irq_data *data)
{
- unsigned int bit = irq - OCTEON_IRQ_SW0;
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+ unsigned int bit = cd->bit;
+
/*
* We don't need to disable IRQs to make these atomic since
* they are already disabled earlier in the low level
clear_c0_cause(0x100 << bit);
}
-static void octeon_irq_core_eoi(unsigned int irq)
+static void octeon_irq_core_eoi(struct irq_data *data)
{
- struct irq_desc *desc = irq_to_desc(irq);
- unsigned int bit = irq - OCTEON_IRQ_SW0;
- /*
- * If an IRQ is being processed while we are disabling it the
- * handler will attempt to unmask the interrupt after it has
- * been disabled.
- */
- if ((unlikely(desc->status & IRQ_DISABLED)))
- return;
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+
/*
* We don't need to disable IRQs to make these atomic since
* they are already disabled earlier in the low level
* interrupt code.
*/
- set_c0_status(0x100 << bit);
+ set_c0_status(0x100 << cd->bit);
}
-static void octeon_irq_core_enable(unsigned int irq)
+static void octeon_irq_core_set_enable_local(void *arg)
{
- unsigned long flags;
- unsigned int bit = irq - OCTEON_IRQ_SW0;
+ struct irq_data *data = arg;
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+ unsigned int mask = 0x100 << cd->bit;
/*
- * We need to disable interrupts to make sure our updates are
- * atomic.
+ * Interrupts are already disabled, so these are atomic.
*/
- local_irq_save(flags);
- set_c0_status(0x100 << bit);
- local_irq_restore(flags);
+ if (cd->desired_en)
+ set_c0_status(mask);
+ else
+ clear_c0_status(mask);
+
}
-static void octeon_irq_core_disable_local(unsigned int irq)
+static void octeon_irq_core_disable(struct irq_data *data)
{
- unsigned long flags;
- unsigned int bit = irq - OCTEON_IRQ_SW0;
- /*
- * We need to disable interrupts to make sure our updates are
- * atomic.
- */
- local_irq_save(flags);
- clear_c0_status(0x100 << bit);
- local_irq_restore(flags);
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+ cd->desired_en = false;
}
-static void octeon_irq_core_disable(unsigned int irq)
+static void octeon_irq_core_enable(struct irq_data *data)
{
-#ifdef CONFIG_SMP
- on_each_cpu((void (*)(void *)) octeon_irq_core_disable_local,
- (void *) (long) irq, 1);
-#else
- octeon_irq_core_disable_local(irq);
-#endif
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+ cd->desired_en = true;
}
-static struct irq_chip octeon_irq_chip_core = {
- .name = "Core",
- .enable = octeon_irq_core_enable,
- .disable = octeon_irq_core_disable,
- .ack = octeon_irq_core_ack,
- .eoi = octeon_irq_core_eoi,
-};
+static void octeon_irq_core_bus_lock(struct irq_data *data)
+{
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+ mutex_lock(&cd->core_irq_mutex);
+}
-static void octeon_irq_ciu0_ack(unsigned int irq)
+static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)
{
- switch (irq) {
- case OCTEON_IRQ_GMX_DRP0:
- case OCTEON_IRQ_GMX_DRP1:
- case OCTEON_IRQ_IPD_DRP:
- case OCTEON_IRQ_KEY_ZERO:
- case OCTEON_IRQ_TIMER0:
- case OCTEON_IRQ_TIMER1:
- case OCTEON_IRQ_TIMER2:
- case OCTEON_IRQ_TIMER3:
- {
- int index = cvmx_get_core_num() * 2;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
- /*
- * CIU timer type interrupts must be acknoleged by
- * writing a '1' bit to their sum0 bit.
- */
- cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
- break;
- }
- default:
- break;
+ struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
+
+ if (cd->desired_en != cd->current_en) {
+ on_each_cpu(octeon_irq_core_set_enable_local, data, 1);
+
+ cd->current_en = cd->desired_en;
}
- /*
- * In order to avoid any locking accessing the CIU, we
- * acknowledge CIU interrupts by disabling all of them. This
- * way we can use a per core register and avoid any out of
- * core locking requirements. This has the side affect that
- * CIU interrupts can't be processed recursively.
- *
- * We don't need to disable IRQs to make these atomic since
- * they are already disabled earlier in the low level
- * interrupt code.
- */
- clear_c0_status(0x100 << 2);
+ mutex_unlock(&cd->core_irq_mutex);
}
-static void octeon_irq_ciu0_eoi(unsigned int irq)
+
+static void octeon_irq_core_cpu_online(struct irq_data *data)
{
- /*
- * Enable all CIU interrupts again. We don't need to disable
- * IRQs to make these atomic since they are already disabled
- * earlier in the low level interrupt code.
- */
- set_c0_status(0x100 << 2);
+ if (irqd_irq_disabled(data))
+ octeon_irq_core_eoi(data);
+}
+
+static void octeon_irq_core_cpu_offline(struct irq_data *data)
+{
+ if (irqd_irq_disabled(data))
+ octeon_irq_core_ack(data);
}
-static int next_coreid_for_irq(struct irq_desc *desc)
+static struct irq_chip octeon_irq_chip_core = {
+ .name = "Core",
+ .irq_enable = octeon_irq_core_enable,
+ .irq_disable = octeon_irq_core_disable,
+ .irq_ack = octeon_irq_core_ack,
+ .irq_eoi = octeon_irq_core_eoi,
+ .irq_bus_lock = octeon_irq_core_bus_lock,
+ .irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,
+
+ .irq_cpu_online = octeon_irq_core_cpu_online,
+ .irq_cpu_offline = octeon_irq_core_cpu_offline,
+};
+
+static void __init octeon_irq_init_core(void)
+{
+ int i;
+ int irq;
+ struct octeon_core_chip_data *cd;
+
+ for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {
+ cd = &octeon_irq_core_chip_data[i];
+ cd->current_en = false;
+ cd->desired_en = false;
+ cd->bit = i;
+ mutex_init(&cd->core_irq_mutex);
+
+ irq = OCTEON_IRQ_SW0 + i;
+ switch (irq) {
+ case OCTEON_IRQ_TIMER:
+ case OCTEON_IRQ_SW0:
+ case OCTEON_IRQ_SW1:
+ case OCTEON_IRQ_5:
+ case OCTEON_IRQ_PERF:
+ irq_set_chip_data(irq, cd);
+ irq_set_chip_and_handler(irq, &octeon_irq_chip_core,
+ handle_percpu_irq);
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+static int next_cpu_for_irq(struct irq_data *data)
{
#ifdef CONFIG_SMP
- int coreid;
- int weight = cpumask_weight(desc->affinity);
+ int cpu;
+ int weight = cpumask_weight(data->affinity);
if (weight > 1) {
- int cpu = smp_processor_id();
+ cpu = smp_processor_id();
for (;;) {
- cpu = cpumask_next(cpu, desc->affinity);
+ cpu = cpumask_next(cpu, data->affinity);
if (cpu >= nr_cpu_ids) {
cpu = -1;
continue;
break;
}
}
- coreid = octeon_coreid_for_cpu(cpu);
} else if (weight == 1) {
- coreid = octeon_coreid_for_cpu(cpumask_first(desc->affinity));
+ cpu = cpumask_first(data->affinity);
} else {
- coreid = cvmx_get_core_num();
+ cpu = smp_processor_id();
}
- return coreid;
+ return cpu;
#else
- return cvmx_get_core_num();
+ return smp_processor_id();
#endif
}
-static void octeon_irq_ciu0_enable(unsigned int irq)
+static void octeon_irq_ciu_enable(struct irq_data *data)
{
- struct irq_desc *desc = irq_to_desc(irq);
- int coreid = next_coreid_for_irq(desc);
+ int cpu = next_cpu_for_irq(data);
+ int coreid = octeon_coreid_for_cpu(cpu);
+ unsigned long *pen;
unsigned long flags;
- uint64_t en0;
- int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */
+ union octeon_ciu_chip_data cd;
+
+ cd.p = irq_data_get_irq_chip_data(data);
- raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
- en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
- en0 |= 1ull << bit;
- cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
- cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ if (cd.s.line == 0) {
+ raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
+ pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ } else {
+ raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+ pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+ }
}
-static void octeon_irq_ciu0_enable_mbox(unsigned int irq)
+static void octeon_irq_ciu_enable_local(struct irq_data *data)
+{
+ unsigned long *pen;
+ unsigned long flags;
+ union octeon_ciu_chip_data cd;
+
+ cd.p = irq_data_get_irq_chip_data(data);
+
+ if (cd.s.line == 0) {
+ raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
+ pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ } else {
+ raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+ pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+ }
+}
+
+static void octeon_irq_ciu_disable_local(struct irq_data *data)
+{
+ unsigned long *pen;
+ unsigned long flags;
+ union octeon_ciu_chip_data cd;
+
+ cd.p = irq_data_get_irq_chip_data(data);
+
+ if (cd.s.line == 0) {
+ raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
+ pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
+ clear_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ } else {
+ raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+ pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
+ clear_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+ }
+}
+
+static void octeon_irq_ciu_disable_all(struct irq_data *data)
{
- int coreid = cvmx_get_core_num();
unsigned long flags;
- uint64_t en0;
- int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */
+ unsigned long *pen;
+ int cpu;
+ union octeon_ciu_chip_data cd;
+
+ wmb(); /* Make sure flag changes arrive before register updates. */
- raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
- en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
- en0 |= 1ull << bit;
- cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
- cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ cd.p = irq_data_get_irq_chip_data(data);
+
+ if (cd.s.line == 0) {
+ raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
+ for_each_online_cpu(cpu) {
+ int coreid = octeon_coreid_for_cpu(cpu);
+ pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
+ clear_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+ }
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ } else {
+ raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+ for_each_online_cpu(cpu) {
+ int coreid = octeon_coreid_for_cpu(cpu);
+ pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+ clear_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
+ }
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+ }
}
-static void octeon_irq_ciu0_disable(unsigned int irq)
+static void octeon_irq_ciu_enable_all(struct irq_data *data)
{
- int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */
unsigned long flags;
- uint64_t en0;
+ unsigned long *pen;
int cpu;
- raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
- for_each_online_cpu(cpu) {
- int coreid = octeon_coreid_for_cpu(cpu);
- en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
- en0 &= ~(1ull << bit);
- cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
+ union octeon_ciu_chip_data cd;
+
+ cd.p = irq_data_get_irq_chip_data(data);
+
+ if (cd.s.line == 0) {
+ raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
+ for_each_online_cpu(cpu) {
+ int coreid = octeon_coreid_for_cpu(cpu);
+ pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+ }
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ } else {
+ raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+ for_each_online_cpu(cpu) {
+ int coreid = octeon_coreid_for_cpu(cpu);
+ pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
+ }
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
}
- /*
- * We need to do a read after the last update to make sure all
- * of them are done.
- */
- cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
}
/*
* Enable the irq on the next core in the affinity set for chips that
* have the EN*_W1{S,C} registers.
*/
-static void octeon_irq_ciu0_enable_v2(unsigned int irq)
+static void octeon_irq_ciu_enable_v2(struct irq_data *data)
{
- int index;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
- struct irq_desc *desc = irq_to_desc(irq);
+ u64 mask;
+ int cpu = next_cpu_for_irq(data);
+ union octeon_ciu_chip_data cd;
- if ((desc->status & IRQ_DISABLED) == 0) {
- index = next_coreid_for_irq(desc) * 2;
+ cd.p = irq_data_get_irq_chip_data(data);
+ mask = 1ull << (cd.s.bit);
+
+ /*
+ * Called under the desc lock, so these should never get out
+ * of sync.
+ */
+ if (cd.s.line == 0) {
+ int index = octeon_coreid_for_cpu(cpu) * 2;
+ set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
+ } else {
+ int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
+ set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
}
}
* Enable the irq on the current CPU for chips that
* have the EN*_W1{S,C} registers.
*/
-static void octeon_irq_ciu0_enable_mbox_v2(unsigned int irq)
+static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)
{
- int index;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
+ u64 mask;
+ union octeon_ciu_chip_data cd;
+
+ cd.p = irq_data_get_irq_chip_data(data);
+ mask = 1ull << (cd.s.bit);
- index = cvmx_get_core_num() * 2;
- cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
+ if (cd.s.line == 0) {
+ int index = cvmx_get_core_num() * 2;
+ set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));
+ cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
+ } else {
+ int index = cvmx_get_core_num() * 2 + 1;
+ set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
+ }
+}
+
+static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)
+{
+ u64 mask;
+ union octeon_ciu_chip_data cd;
+
+ cd.p = irq_data_get_irq_chip_data(data);
+ mask = 1ull << (cd.s.bit);
+
+ if (cd.s.line == 0) {
+ int index = cvmx_get_core_num() * 2;
+ clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));
+ cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
+ } else {
+ int index = cvmx_get_core_num() * 2 + 1;
+ clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
+ }
}
/*
- * Disable the irq on the current core for chips that have the EN*_W1{S,C}
- * registers.
+ * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
*/
-static void octeon_irq_ciu0_ack_v2(unsigned int irq)
-{
- int index = cvmx_get_core_num() * 2;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
-
- switch (irq) {
- case OCTEON_IRQ_GMX_DRP0:
- case OCTEON_IRQ_GMX_DRP1:
- case OCTEON_IRQ_IPD_DRP:
- case OCTEON_IRQ_KEY_ZERO:
- case OCTEON_IRQ_TIMER0:
- case OCTEON_IRQ_TIMER1:
- case OCTEON_IRQ_TIMER2:
- case OCTEON_IRQ_TIMER3:
- /*
- * CIU timer type interrupts must be acknoleged by
- * writing a '1' bit to their sum0 bit.
- */
+static void octeon_irq_ciu_ack(struct irq_data *data)
+{
+ u64 mask;
+ union octeon_ciu_chip_data cd;
+
+ cd.p = data->chip_data;
+ mask = 1ull << (cd.s.bit);
+
+ if (cd.s.line == 0) {
+ int index = cvmx_get_core_num() * 2;
cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
- break;
- default:
- break;
+ } else {
+ cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);
}
-
- cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
}
/*
- * Enable the irq on the current core for chips that have the EN*_W1{S,C}
+ * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
* registers.
*/
-static void octeon_irq_ciu0_eoi_mbox_v2(unsigned int irq)
+static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
{
- struct irq_desc *desc = irq_to_desc(irq);
- int index = cvmx_get_core_num() * 2;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
+ int cpu;
+ u64 mask;
+ union octeon_ciu_chip_data cd;
- if (likely((desc->status & IRQ_DISABLED) == 0))
- cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
+ wmb(); /* Make sure flag changes arrive before register updates. */
+
+ cd.p = data->chip_data;
+ mask = 1ull << (cd.s.bit);
+
+ if (cd.s.line == 0) {
+ for_each_online_cpu(cpu) {
+ int index = octeon_coreid_for_cpu(cpu) * 2;
+ clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
+ cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
+ }
+ } else {
+ for_each_online_cpu(cpu) {
+ int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
+ clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
+ }
+ }
}
/*
- * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
+ * Enable the irq on the all cores for chips that have the EN*_W1{S,C}
* registers.
*/
-static void octeon_irq_ciu0_disable_all_v2(unsigned int irq)
+static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)
{
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
- int index;
int cpu;
- for_each_online_cpu(cpu) {
- index = octeon_coreid_for_cpu(cpu) * 2;
- cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
+ u64 mask;
+ union octeon_ciu_chip_data cd;
+
+ cd.p = data->chip_data;
+ mask = 1ull << (cd.s.bit);
+
+ if (cd.s.line == 0) {
+ for_each_online_cpu(cpu) {
+ int index = octeon_coreid_for_cpu(cpu) * 2;
+ set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
+ cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
+ }
+ } else {
+ for_each_online_cpu(cpu) {
+ int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
+ set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
+ }
}
}
+static void octeon_irq_cpu_online_mbox(struct irq_data *data)
+{
+ if (irqd_irq_disabled(data))
+ octeon_irq_ciu_enable_local(data);
+}
+
+static void octeon_irq_cpu_online_mbox_v2(struct irq_data *data)
+{
+ if (irqd_irq_disabled(data))
+ octeon_irq_ciu_enable_local_v2(data);
+}
+
+static void octeon_irq_cpu_offline_mbox(struct irq_data *data)
+{
+ if (irqd_irq_disabled(data))
+ octeon_irq_ciu_disable_local(data);
+}
+
+static void octeon_irq_cpu_offline_mbox_v2(struct irq_data *data)
+{
+ if (irqd_irq_disabled(data))
+ octeon_irq_ciu_disable_local_v2(data);
+}
+
#ifdef CONFIG_SMP
-static int octeon_irq_ciu0_set_affinity(unsigned int irq, const struct cpumask *dest)
+
+static void octeon_irq_cpu_offline_ciu(struct irq_data *data)
+{
+ int cpu = smp_processor_id();
+ cpumask_t new_affinity;
+
+ if (!cpumask_test_cpu(cpu, data->affinity))
+ return;
+
+ if (cpumask_weight(data->affinity) > 1) {
+ /*
+ * It has multi CPU affinity, just remove this CPU
+ * from the affinity set.
+ */
+ cpumask_copy(&new_affinity, data->affinity);
+ cpumask_clear_cpu(cpu, &new_affinity);
+ } else {
+ /* Otherwise, put it on lowest numbered online CPU. */
+ cpumask_clear(&new_affinity);
+ cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
+ }
+ __irq_set_affinity_locked(data, &new_affinity);
+}
+
+static int octeon_irq_ciu_set_affinity(struct irq_data *data,
+ const struct cpumask *dest, bool force)
{
int cpu;
- struct irq_desc *desc = irq_to_desc(irq);
+ struct irq_desc *desc = irq_to_desc(data->irq);
int enable_one = (desc->status & IRQ_DISABLED) == 0;
unsigned long flags;
- int bit = irq - OCTEON_IRQ_WORKQ0; /* Bit 0-63 of EN0 */
+ union octeon_ciu_chip_data cd;
+
+ cd.p = data->chip_data;
/*
* For non-v2 CIU, we will allow only single CPU affinity.
if (cpumask_weight(dest) != 1)
return -EINVAL;
- raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
- for_each_online_cpu(cpu) {
- int coreid = octeon_coreid_for_cpu(cpu);
- uint64_t en0 =
- cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
- if (cpumask_test_cpu(cpu, dest) && enable_one) {
- enable_one = 0;
- en0 |= 1ull << bit;
- } else {
- en0 &= ~(1ull << bit);
+ if (desc->status & IRQ_DISABLED)
+ return 0;
+
+ if (cd.s.line == 0) {
+ raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
+ for_each_online_cpu(cpu) {
+ int coreid = octeon_coreid_for_cpu(cpu);
+ unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
+
+ if (cpumask_test_cpu(cpu, dest) && enable_one) {
+ enable_one = 0;
+ set_bit(cd.s.bit, pen);
+ } else {
+ clear_bit(cd.s.bit, pen);
+ }
+ cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+ }
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
+ } else {
+ raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+ for_each_online_cpu(cpu) {
+ int coreid = octeon_coreid_for_cpu(cpu);
+ unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+
+ if (cpumask_test_cpu(cpu, dest) && enable_one) {
+ enable_one = 0;
+ set_bit(cd.s.bit, pen);
+ } else {
+ clear_bit(cd.s.bit, pen);
+ }
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
}
- cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
+ raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
}
- /*
- * We need to do a read after the last update to make sure all
- * of them are done.
- */
- cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
-
return 0;
}
* Set affinity for the irq for chips that have the EN*_W1{S,C}
* registers.
*/
-static int octeon_irq_ciu0_set_affinity_v2(unsigned int irq,
- const struct cpumask *dest)
+static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
+ const struct cpumask *dest,
+ bool force)
{
int cpu;
- int index;
- struct irq_desc *desc = irq_to_desc(irq);
+ struct irq_desc *desc = irq_to_desc(data->irq);
int enable_one = (desc->status & IRQ_DISABLED) == 0;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
-
- for_each_online_cpu(cpu) {
- index = octeon_coreid_for_cpu(cpu) * 2;
- if (cpumask_test_cpu(cpu, dest) && enable_one) {
- enable_one = 0;
- cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
- } else {
- cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
+ u64 mask;
+ union octeon_ciu_chip_data cd;
+
+ if (desc->status & IRQ_DISABLED)
+ return 0;
+
+ cd.p = data->chip_data;
+ mask = 1ull << cd.s.bit;
+
+ if (cd.s.line == 0) {
+ for_each_online_cpu(cpu) {
+ unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
+ int index = octeon_coreid_for_cpu(cpu) * 2;
+ if (cpumask_test_cpu(cpu, dest) && enable_one) {
+ enable_one = 0;
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
+ } else {
+ clear_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
+ }
+ }
+ } else {
+ for_each_online_cpu(cpu) {
+ unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+ int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
+ if (cpumask_test_cpu(cpu, dest) && enable_one) {
+ enable_one = 0;
+ set_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
+ } else {
+ clear_bit(cd.s.bit, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
+ }
}
}
return 0;
}
#endif
+/*
+ * The v1 CIU code already masks things, so supply a dummy version to
+ * the core chip code.
+ */
+static void octeon_irq_dummy_mask(struct irq_data *data)
+{
+ return;
+}
+
/*
* Newer octeon chips have support for lockless CIU operation.
*/
-static struct irq_chip octeon_irq_chip_ciu0_v2 = {
- .name = "CIU0",
- .enable = octeon_irq_ciu0_enable_v2,
- .disable = octeon_irq_ciu0_disable_all_v2,
- .eoi = octeon_irq_ciu0_enable_v2,
+static struct irq_chip octeon_irq_chip_ciu_v2 = {
+ .name = "CIU",
+ .irq_enable = octeon_irq_ciu_enable_v2,
+ .irq_disable = octeon_irq_ciu_disable_all_v2,
+ .irq_mask = octeon_irq_ciu_disable_local_v2,
+ .irq_unmask = octeon_irq_ciu_enable_v2,
#ifdef CONFIG_SMP
- .set_affinity = octeon_irq_ciu0_set_affinity_v2,
+ .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
+ .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
#endif
};
-static struct irq_chip octeon_irq_chip_ciu0 = {
- .name = "CIU0",
- .enable = octeon_irq_ciu0_enable,
- .disable = octeon_irq_ciu0_disable,
- .eoi = octeon_irq_ciu0_eoi,
+static struct irq_chip octeon_irq_chip_ciu_edge_v2 = {
+ .name = "CIU-E",
+ .irq_enable = octeon_irq_ciu_enable_v2,
+ .irq_disable = octeon_irq_ciu_disable_all_v2,
+ .irq_ack = octeon_irq_ciu_ack,
+ .irq_mask = octeon_irq_ciu_disable_local_v2,
+ .irq_unmask = octeon_irq_ciu_enable_v2,
#ifdef CONFIG_SMP
- .set_affinity = octeon_irq_ciu0_set_affinity,
+ .irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
+ .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
#endif
};
-/* The mbox versions don't do any affinity or round-robin. */
-static struct irq_chip octeon_irq_chip_ciu0_mbox_v2 = {
- .name = "CIU0-M",
- .enable = octeon_irq_ciu0_enable_mbox_v2,
- .disable = octeon_irq_ciu0_disable,
- .eoi = octeon_irq_ciu0_eoi_mbox_v2,
+static struct irq_chip octeon_irq_chip_ciu = {
+ .name = "CIU",
+ .irq_enable = octeon_irq_ciu_enable,
+ .irq_disable = octeon_irq_ciu_disable_all,
+ .irq_mask = octeon_irq_dummy_mask,
+#ifdef CONFIG_SMP
+ .irq_set_affinity = octeon_irq_ciu_set_affinity,
+ .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
+#endif
};
-static struct irq_chip octeon_irq_chip_ciu0_mbox = {
- .name = "CIU0-M",
- .enable = octeon_irq_ciu0_enable_mbox,
- .disable = octeon_irq_ciu0_disable,
- .eoi = octeon_irq_ciu0_eoi,
+static struct irq_chip octeon_irq_chip_ciu_edge = {
+ .name = "CIU-E",
+ .irq_enable = octeon_irq_ciu_enable,
+ .irq_disable = octeon_irq_ciu_disable_all,
+ .irq_mask = octeon_irq_dummy_mask,
+ .irq_ack = octeon_irq_ciu_ack,
+#ifdef CONFIG_SMP
+ .irq_set_affinity = octeon_irq_ciu_set_affinity,
+ .irq_cpu_offline = octeon_irq_cpu_offline_ciu,
+#endif
};
-static void octeon_irq_ciu1_ack(unsigned int irq)
-{
- /*
- * In order to avoid any locking accessing the CIU, we
- * acknowledge CIU interrupts by disabling all of them. This
- * way we can use a per core register and avoid any out of
- * core locking requirements. This has the side affect that
- * CIU interrupts can't be processed recursively. We don't
- * need to disable IRQs to make these atomic since they are
- * already disabled earlier in the low level interrupt code.
- */
- clear_c0_status(0x100 << 3);
-}
+/* The mbox versions don't do any affinity or round-robin. */
+static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {
+ .name = "CIU-M",
+ .irq_enable = octeon_irq_ciu_enable_all_v2,
+ .irq_disable = octeon_irq_ciu_disable_all_v2,
+ .irq_ack = octeon_irq_ciu_disable_local_v2,
+ .irq_eoi = octeon_irq_ciu_enable_local_v2,
+
+ .irq_cpu_online = octeon_irq_cpu_online_mbox_v2,
+ .irq_cpu_offline = octeon_irq_cpu_offline_mbox_v2,
+};
-static void octeon_irq_ciu1_eoi(unsigned int irq)
-{
- /*
- * Enable all CIU interrupts again. We don't need to disable
- * IRQs to make these atomic since they are already disabled
- * earlier in the low level interrupt code.
- */
- set_c0_status(0x100 << 3);
-}
+static struct irq_chip octeon_irq_chip_ciu_mbox = {
+ .name = "CIU-M",
+ .irq_enable = octeon_irq_ciu_enable_all,
+ .irq_disable = octeon_irq_ciu_disable_all,
-static void octeon_irq_ciu1_enable(unsigned int irq)
+ .irq_cpu_online = octeon_irq_cpu_online_mbox,
+ .irq_cpu_offline = octeon_irq_cpu_offline_mbox,
+};
+
+/*
+ * Watchdog interrupts are special. They are associated with a single
+ * core, so we hardwire the affinity to that core.
+ */
+static void octeon_irq_ciu_wd_enable(struct irq_data *data)
{
- struct irq_desc *desc = irq_to_desc(irq);
- int coreid = next_coreid_for_irq(desc);
unsigned long flags;
- uint64_t en1;
- int bit = irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */
+ unsigned long *pen;
+ int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */
+ int cpu = octeon_cpu_for_coreid(coreid);
raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
- en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
- en1 |= 1ull << bit;
- cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
- cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
+ pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+ set_bit(coreid, pen);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
}
* Watchdog interrupts are special. They are associated with a single
* core, so we hardwire the affinity to that core.
*/
-static void octeon_irq_ciu1_wd_enable(unsigned int irq)
+static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)
{
- unsigned long flags;
- uint64_t en1;
- int bit = irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */
- int coreid = bit;
+ int coreid = data->irq - OCTEON_IRQ_WDOG0;
+ int cpu = octeon_cpu_for_coreid(coreid);
- raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
- en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
- en1 |= 1ull << bit;
- cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
- cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+ set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
+ cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);
}
-static void octeon_irq_ciu1_disable(unsigned int irq)
+
+static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {
+ .name = "CIU-W",
+ .irq_enable = octeon_irq_ciu1_wd_enable_v2,
+ .irq_disable = octeon_irq_ciu_disable_all_v2,
+ .irq_mask = octeon_irq_ciu_disable_local_v2,
+ .irq_unmask = octeon_irq_ciu_enable_local_v2,
+};
+
+static struct irq_chip octeon_irq_chip_ciu_wd = {
+ .name = "CIU-W",
+ .irq_enable = octeon_irq_ciu_wd_enable,
+ .irq_disable = octeon_irq_ciu_disable_all,
+ .irq_mask = octeon_irq_dummy_mask,
+};
+
+static void octeon_irq_ip2_v1(void)
{
- int bit = irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */
- unsigned long flags;
- uint64_t en1;
- int cpu;
- raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
- for_each_online_cpu(cpu) {
- int coreid = octeon_coreid_for_cpu(cpu);
- en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
- en1 &= ~(1ull << bit);
- cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
+ const unsigned long core_id = cvmx_get_core_num();
+ u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
+
+ ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);
+ clear_c0_status(STATUSF_IP2);
+ if (likely(ciu_sum)) {
+ int bit = fls64(ciu_sum) - 1;
+ int irq = octeon_irq_ciu_to_irq[0][bit];
+ if (likely(irq))
+ do_IRQ(irq);
+ else
+ spurious_interrupt();
+ } else {
+ spurious_interrupt();
}
- /*
- * We need to do a read after the last update to make sure all
- * of them are done.
- */
- cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+ set_c0_status(STATUSF_IP2);
}
-/*
- * Enable the irq on the current core for chips that have the EN*_W1{S,C}
- * registers.
- */
-static void octeon_irq_ciu1_enable_v2(unsigned int irq)
+static void octeon_irq_ip2_v2(void)
{
- int index;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
- struct irq_desc *desc = irq_to_desc(irq);
-
- if ((desc->status & IRQ_DISABLED) == 0) {
- index = next_coreid_for_irq(desc) * 2 + 1;
- cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
+ const unsigned long core_id = cvmx_get_core_num();
+ u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
+
+ ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);
+ if (likely(ciu_sum)) {
+ int bit = fls64(ciu_sum) - 1;
+ int irq = octeon_irq_ciu_to_irq[0][bit];
+ if (likely(irq))
+ do_IRQ(irq);
+ else
+ spurious_interrupt();
+ } else {
+ spurious_interrupt();
}
}
-
-/*
- * Watchdog interrupts are special. They are associated with a single
- * core, so we hardwire the affinity to that core.
- */
-static void octeon_irq_ciu1_wd_enable_v2(unsigned int irq)
+static void octeon_irq_ip3_v1(void)
{
- int index;
- int coreid = irq - OCTEON_IRQ_WDOG0;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
- struct irq_desc *desc = irq_to_desc(irq);
-
- if ((desc->status & IRQ_DISABLED) == 0) {
- index = coreid * 2 + 1;
- cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
+ u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
+
+ ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);
+ clear_c0_status(STATUSF_IP3);
+ if (likely(ciu_sum)) {
+ int bit = fls64(ciu_sum) - 1;
+ int irq = octeon_irq_ciu_to_irq[1][bit];
+ if (likely(irq))
+ do_IRQ(irq);
+ else
+ spurious_interrupt();
+ } else {
+ spurious_interrupt();
}
+ set_c0_status(STATUSF_IP3);
}
-/*
- * Disable the irq on the current core for chips that have the EN*_W1{S,C}
- * registers.
- */
-static void octeon_irq_ciu1_ack_v2(unsigned int irq)
+static void octeon_irq_ip3_v2(void)
{
- int index = cvmx_get_core_num() * 2 + 1;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
-
- cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
+ u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
+
+ ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);
+ if (likely(ciu_sum)) {
+ int bit = fls64(ciu_sum) - 1;
+ int irq = octeon_irq_ciu_to_irq[1][bit];
+ if (likely(irq))
+ do_IRQ(irq);
+ else
+ spurious_interrupt();
+ } else {
+ spurious_interrupt();
+ }
}
-/*
- * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
- * registers.
- */
-static void octeon_irq_ciu1_disable_all_v2(unsigned int irq)
+static void octeon_irq_ip4_mask(void)
{
- u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
- int index;
- int cpu;
- for_each_online_cpu(cpu) {
- index = octeon_coreid_for_cpu(cpu) * 2 + 1;
- cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
- }
+ clear_c0_status(STATUSF_IP4);
+ spurious_interrupt();
}
-#ifdef CONFIG_SMP
-static int octeon_irq_ciu1_set_affinity(unsigned int irq,
- const struct cpumask *dest)
-{
- int cpu;
- struct irq_desc *desc = irq_to_desc(irq);
- int enable_one = (desc->status & IRQ_DISABLED) == 0;
- unsigned long flags;
- int bit = irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */
+static void (*octeon_irq_ip2)(void);
+static void (*octeon_irq_ip3)(void);
+static void (*octeon_irq_ip4)(void);
- /*
- * For non-v2 CIU, we will allow only single CPU affinity.
- * This removes the need to do locking in the .ack/.eoi
- * functions.
- */
- if (cpumask_weight(dest) != 1)
- return -EINVAL;
+void __cpuinitdata (*octeon_irq_setup_secondary)(void);
- raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
- for_each_online_cpu(cpu) {
- int coreid = octeon_coreid_for_cpu(cpu);
- uint64_t en1 =
- cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
- if (cpumask_test_cpu(cpu, dest) && enable_one) {
- enable_one = 0;
- en1 |= 1ull << bit;
- } else {
- en1 &= ~(1ull << bit);
- }
- cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
- }
+static void __cpuinit octeon_irq_percpu_enable(void)
+{
+ irq_cpu_online();
+}
+
+static void __cpuinit octeon_irq_init_ciu_percpu(void)
+{
+ int coreid = cvmx_get_core_num();
/*
- * We need to do a read after the last update to make sure all
- * of them are done.
+ * Disable All CIU Interrupts. The ones we need will be
+ * enabled later. Read the SUM register so we know the write
+ * completed.
*/
- cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1));
- raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
-
- return 0;
+ cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);
+ cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);
+ cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);
+ cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));
}
-/*
- * Set affinity for the irq for chips that have the EN*_W1{S,C}
- * registers.
- */
-static int octeon_irq_ciu1_set_affinity_v2(unsigned int irq,
- const struct cpumask *dest)
+static void __cpuinit octeon_irq_setup_secondary_ciu(void)
{
- int cpu;
- int index;
- struct irq_desc *desc = irq_to_desc(irq);
- int enable_one = (desc->status & IRQ_DISABLED) == 0;
- u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
- for_each_online_cpu(cpu) {
- index = octeon_coreid_for_cpu(cpu) * 2 + 1;
- if (cpumask_test_cpu(cpu, dest) && enable_one) {
- enable_one = 0;
- cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
- } else {
- cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
- }
- }
- return 0;
-}
-#endif
-/*
- * Newer octeon chips have support for lockless CIU operation.
- */
-static struct irq_chip octeon_irq_chip_ciu1_v2 = {
- .name = "CIU1",
- .enable = octeon_irq_ciu1_enable_v2,
- .disable = octeon_irq_ciu1_disable_all_v2,
- .eoi = octeon_irq_ciu1_enable_v2,
-#ifdef CONFIG_SMP
- .set_affinity = octeon_irq_ciu1_set_affinity_v2,
-#endif
-};
+ __get_cpu_var(octeon_irq_ciu0_en_mirror) = 0;
+ __get_cpu_var(octeon_irq_ciu1_en_mirror) = 0;
-static struct irq_chip octeon_irq_chip_ciu1 = {
- .name = "CIU1",
- .enable = octeon_irq_ciu1_enable,
- .disable = octeon_irq_ciu1_disable,
- .eoi = octeon_irq_ciu1_eoi,
-#ifdef CONFIG_SMP
- .set_affinity = octeon_irq_ciu1_set_affinity,
-#endif
-};
+ octeon_irq_init_ciu_percpu();
+ octeon_irq_percpu_enable();
-static struct irq_chip octeon_irq_chip_ciu1_wd_v2 = {
- .name = "CIU1-W",
- .enable = octeon_irq_ciu1_wd_enable_v2,
- .disable = octeon_irq_ciu1_disable_all_v2,
- .eoi = octeon_irq_ciu1_wd_enable_v2,
-};
+ /* Enable the CIU lines */
+ set_c0_status(STATUSF_IP3 | STATUSF_IP2);
+ clear_c0_status(STATUSF_IP4);
+}
-static struct irq_chip octeon_irq_chip_ciu1_wd = {
- .name = "CIU1-W",
- .enable = octeon_irq_ciu1_wd_enable,
- .disable = octeon_irq_ciu1_disable,
- .eoi = octeon_irq_ciu1_eoi,
-};
+static void __init octeon_irq_init_ciu(void)
+{
+ unsigned int i;
+ struct irq_chip *chip;
+ struct irq_chip *chip_edge;
+ struct irq_chip *chip_mbox;
+ struct irq_chip *chip_wd;
+
+ octeon_irq_init_ciu_percpu();
+ octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
-static void (*octeon_ciu0_ack)(unsigned int);
-static void (*octeon_ciu1_ack)(unsigned int);
+ if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
+ OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
+ OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) ||
+ OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
+ octeon_irq_ip2 = octeon_irq_ip2_v2;
+ octeon_irq_ip3 = octeon_irq_ip3_v2;
+ chip = &octeon_irq_chip_ciu_v2;
+ chip_edge = &octeon_irq_chip_ciu_edge_v2;
+ chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
+ chip_wd = &octeon_irq_chip_ciu_wd_v2;
+ } else {
+ octeon_irq_ip2 = octeon_irq_ip2_v1;
+ octeon_irq_ip3 = octeon_irq_ip3_v1;
+ chip = &octeon_irq_chip_ciu;
+ chip_edge = &octeon_irq_chip_ciu_edge;
+ chip_mbox = &octeon_irq_chip_ciu_mbox;
+ chip_wd = &octeon_irq_chip_ciu_wd;
+ }
+ octeon_irq_ip4 = octeon_irq_ip4_mask;
+
+ /* Mips internal */
+ octeon_irq_init_core();
+
+ /* CIU_0 */
+ for (i = 0; i < 16; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WORKQ0, 0, i + 0, chip, handle_level_irq);
+ for (i = 0; i < 16; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_GPIO0, 0, i + 16, chip, handle_level_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX0, 0, 32, chip_mbox, handle_percpu_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX1, 0, 33, chip_mbox, handle_percpu_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART0, 0, 34, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART1, 0, 35, chip, handle_level_irq);
+
+ for (i = 0; i < 4; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_PCI_INT0, 0, i + 36, chip, handle_level_irq);
+ for (i = 0; i < 4; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_PCI_MSI0, 0, i + 40, chip, handle_level_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_TWSI, 0, 45, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_RML, 0, 46, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_TRACE0, 0, 47, chip, handle_level_irq);
+
+ for (i = 0; i < 2; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_GMX_DRP0, 0, i + 48, chip_edge, handle_edge_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPD_DRP, 0, 50, chip_edge, handle_edge_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_KEY_ZERO, 0, 51, chip_edge, handle_edge_irq);
+
+ for (i = 0; i < 4; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_TIMER0, 0, i + 52, chip_edge, handle_edge_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB0, 0, 56, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_PCM, 0, 57, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_MPI, 0, 58, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_TWSI2, 0, 59, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_POWIQ, 0, 60, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPDPPTHR, 0, 61, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_MII0, 0, 62, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_BOOTDMA, 0, 63, chip, handle_level_irq);
+
+ /* CIU_1 */
+ for (i = 0; i < 16; i++)
+ octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i + 0, chip_wd, handle_level_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_UART2, 1, 16, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB1, 1, 17, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_MII1, 1, 18, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_NAND, 1, 19, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_MIO, 1, 20, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_IOB, 1, 21, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_FPA, 1, 22, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_POW, 1, 23, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_L2C, 1, 24, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_IPD, 1, 25, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_PIP, 1, 26, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_PKO, 1, 27, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_ZIP, 1, 28, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_TIM, 1, 29, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_RAD, 1, 30, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_KEY, 1, 31, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_DFA, 1, 32, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_USBCTL, 1, 33, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_SLI, 1, 34, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_DPI, 1, 35, chip, handle_level_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_AGX0, 1, 36, chip, handle_level_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_AGL, 1, 46, chip, handle_level_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_PTP, 1, 47, chip_edge, handle_edge_irq);
+
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_PEM0, 1, 48, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_PEM1, 1, 49, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_SRIO0, 1, 50, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_SRIO1, 1, 51, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_LMC0, 1, 52, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_DFM, 1, 56, chip, handle_level_irq);
+ octeon_irq_set_ciu_mapping(OCTEON_IRQ_RST, 1, 63, chip, handle_level_irq);
+
+ /* Enable the CIU lines */
+ set_c0_status(STATUSF_IP3 | STATUSF_IP2);
+ clear_c0_status(STATUSF_IP4);
+}
void __init arch_init_irq(void)
{
- unsigned int irq;
- struct irq_chip *chip0;
- struct irq_chip *chip0_mbox;
- struct irq_chip *chip1;
- struct irq_chip *chip1_wd;
-
#ifdef CONFIG_SMP
/* Set the default affinity to the boot cpu. */
cpumask_clear(irq_default_affinity);
cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
#endif
-
- if (NR_IRQS < OCTEON_IRQ_LAST)
- pr_err("octeon_irq_init: NR_IRQS is set too low\n");
-
- if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
- OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
- OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X)) {
- octeon_ciu0_ack = octeon_irq_ciu0_ack_v2;
- octeon_ciu1_ack = octeon_irq_ciu1_ack_v2;
- chip0 = &octeon_irq_chip_ciu0_v2;
- chip0_mbox = &octeon_irq_chip_ciu0_mbox_v2;
- chip1 = &octeon_irq_chip_ciu1_v2;
- chip1_wd = &octeon_irq_chip_ciu1_wd_v2;
- } else {
- octeon_ciu0_ack = octeon_irq_ciu0_ack;
- octeon_ciu1_ack = octeon_irq_ciu1_ack;
- chip0 = &octeon_irq_chip_ciu0;
- chip0_mbox = &octeon_irq_chip_ciu0_mbox;
- chip1 = &octeon_irq_chip_ciu1;
- chip1_wd = &octeon_irq_chip_ciu1_wd;
- }
-
- /* 0 - 15 reserved for i8259 master and slave controller. */
-
- /* 17 - 23 Mips internal */
- for (irq = OCTEON_IRQ_SW0; irq <= OCTEON_IRQ_TIMER; irq++) {
- set_irq_chip_and_handler(irq, &octeon_irq_chip_core,
- handle_percpu_irq);
- }
-
- /* 24 - 87 CIU_INT_SUM0 */
- for (irq = OCTEON_IRQ_WORKQ0; irq <= OCTEON_IRQ_BOOTDMA; irq++) {
- switch (irq) {
- case OCTEON_IRQ_MBOX0:
- case OCTEON_IRQ_MBOX1:
- set_irq_chip_and_handler(irq, chip0_mbox, handle_percpu_irq);
- break;
- default:
- set_irq_chip_and_handler(irq, chip0, handle_fasteoi_irq);
- break;
- }
- }
-
- /* 88 - 151 CIU_INT_SUM1 */
- for (irq = OCTEON_IRQ_WDOG0; irq <= OCTEON_IRQ_WDOG15; irq++)
- set_irq_chip_and_handler(irq, chip1_wd, handle_fasteoi_irq);
-
- for (irq = OCTEON_IRQ_UART2; irq <= OCTEON_IRQ_RESERVED151; irq++)
- set_irq_chip_and_handler(irq, chip1, handle_fasteoi_irq);
-
- set_c0_status(0x300 << 2);
+ octeon_irq_init_ciu();
}
asmlinkage void plat_irq_dispatch(void)
{
- const unsigned long core_id = cvmx_get_core_num();
- const uint64_t ciu_sum0_address = CVMX_CIU_INTX_SUM0(core_id * 2);
- const uint64_t ciu_en0_address = CVMX_CIU_INTX_EN0(core_id * 2);
- const uint64_t ciu_sum1_address = CVMX_CIU_INT_SUM1;
- const uint64_t ciu_en1_address = CVMX_CIU_INTX_EN1(core_id * 2 + 1);
unsigned long cop0_cause;
unsigned long cop0_status;
- uint64_t ciu_en;
- uint64_t ciu_sum;
- unsigned int irq;
while (1) {
cop0_cause = read_c0_cause();
cop0_cause &= cop0_status;
cop0_cause &= ST0_IM;
- if (unlikely(cop0_cause & STATUSF_IP2)) {
- ciu_sum = cvmx_read_csr(ciu_sum0_address);
- ciu_en = cvmx_read_csr(ciu_en0_address);
- ciu_sum &= ciu_en;
- if (likely(ciu_sum)) {
- irq = fls64(ciu_sum) + OCTEON_IRQ_WORKQ0 - 1;
- octeon_ciu0_ack(irq);
- do_IRQ(irq);
- } else {
- spurious_interrupt();
- }
- } else if (unlikely(cop0_cause & STATUSF_IP3)) {
- ciu_sum = cvmx_read_csr(ciu_sum1_address);
- ciu_en = cvmx_read_csr(ciu_en1_address);
- ciu_sum &= ciu_en;
- if (likely(ciu_sum)) {
- irq = fls64(ciu_sum) + OCTEON_IRQ_WDOG0 - 1;
- octeon_ciu1_ack(irq);
- do_IRQ(irq);
- } else {
- spurious_interrupt();
- }
- } else if (likely(cop0_cause)) {
+ if (unlikely(cop0_cause & STATUSF_IP2))
+ octeon_irq_ip2();
+ else if (unlikely(cop0_cause & STATUSF_IP3))
+ octeon_irq_ip3();
+ else if (unlikely(cop0_cause & STATUSF_IP4))
+ octeon_irq_ip4();
+ else if (likely(cop0_cause))
do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
- } else {
+ else
break;
- }
}
}
void fixup_irqs(void)
{
- int irq;
- struct irq_desc *desc;
- cpumask_t new_affinity;
- unsigned long flags;
- int do_set_affinity;
- int cpu;
-
- cpu = smp_processor_id();
-
- for (irq = OCTEON_IRQ_SW0; irq <= OCTEON_IRQ_TIMER; irq++)
- octeon_irq_core_disable_local(irq);
-
- for (irq = OCTEON_IRQ_WORKQ0; irq < OCTEON_IRQ_LAST; irq++) {
- desc = irq_to_desc(irq);
- switch (irq) {
- case OCTEON_IRQ_MBOX0:
- case OCTEON_IRQ_MBOX1:
- /* The eoi function will disable them on this CPU. */
- desc->chip->eoi(irq);
- break;
- case OCTEON_IRQ_WDOG0:
- case OCTEON_IRQ_WDOG1:
- case OCTEON_IRQ_WDOG2:
- case OCTEON_IRQ_WDOG3:
- case OCTEON_IRQ_WDOG4:
- case OCTEON_IRQ_WDOG5:
- case OCTEON_IRQ_WDOG6:
- case OCTEON_IRQ_WDOG7:
- case OCTEON_IRQ_WDOG8:
- case OCTEON_IRQ_WDOG9:
- case OCTEON_IRQ_WDOG10:
- case OCTEON_IRQ_WDOG11:
- case OCTEON_IRQ_WDOG12:
- case OCTEON_IRQ_WDOG13:
- case OCTEON_IRQ_WDOG14:
- case OCTEON_IRQ_WDOG15:
- /*
- * These have special per CPU semantics and
- * are handled in the watchdog driver.
- */
- break;
- default:
- raw_spin_lock_irqsave(&desc->lock, flags);
- /*
- * If this irq has an action, it is in use and
- * must be migrated if it has affinity to this
- * cpu.
- */
- if (desc->action && cpumask_test_cpu(cpu, desc->affinity)) {
- if (cpumask_weight(desc->affinity) > 1) {
- /*
- * It has multi CPU affinity,
- * just remove this CPU from
- * the affinity set.
- */
- cpumask_copy(&new_affinity, desc->affinity);
- cpumask_clear_cpu(cpu, &new_affinity);
- } else {
- /*
- * Otherwise, put it on lowest
- * numbered online CPU.
- */
- cpumask_clear(&new_affinity);
- cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
- }
- do_set_affinity = 1;
- } else {
- do_set_affinity = 0;
- }
- raw_spin_unlock_irqrestore(&desc->lock, flags);
-
- if (do_set_affinity)
- irq_set_affinity(irq, &new_affinity);
-
- break;
- }
- }
+ irq_cpu_offline();
}
#endif /* CONFIG_HOTPLUG_CPU */
projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / commitdiff