2 * Copyright 2012 Michael Ellerman, IBM Corporation.
3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
10 #include <linux/kernel.h>
11 #include <linux/kvm_host.h>
12 #include <linux/err.h>
13 #include <linux/gfp.h>
14 #include <linux/anon_inodes.h>
16 #include <asm/uaccess.h>
17 #include <asm/kvm_book3s.h>
18 #include <asm/kvm_ppc.h>
19 #include <asm/hvcall.h>
21 #include <asm/debug.h>
23 #include <linux/debugfs.h>
24 #include <linux/seq_file.h>
26 #include "book3s_xics.h"
29 #define XICS_DBG(fmt...) do { } while (0)
31 #define XICS_DBG(fmt...) trace_printk(fmt)
34 #define ENABLE_REALMODE true
35 #define DEBUG_REALMODE false
41 * Each ICS has a mutex protecting the information about the IRQ
42 * sources and avoiding simultaneous deliveries if the same interrupt.
44 * ICP operations are done via a single compare & swap transaction
45 * (most ICP state fits in the union kvmppc_icp_state)
52 * - To speed up resends, keep a bitmap of "resend" set bits in the
55 * - Speed up server# -> ICP lookup (array ? hash table ?)
57 * - Make ICS lockless as well, or at least a per-interrupt lock or hashed
58 * locks array to improve scalability
61 /* -- ICS routines -- */
63 static void icp_deliver_irq(struct kvmppc_xics
*xics
, struct kvmppc_icp
*icp
,
66 static int ics_deliver_irq(struct kvmppc_xics
*xics
, u32 irq
, u32 level
,
69 struct ics_irq_state
*state
;
70 struct kvmppc_ics
*ics
;
73 XICS_DBG("ics deliver %#x (level: %d)\n", irq
, level
);
75 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
77 XICS_DBG("ics_deliver_irq: IRQ 0x%06x not found !\n", irq
);
80 state
= &ics
->irq_state
[src
];
85 return state
->asserted
;
88 * We set state->asserted locklessly. This should be fine as
89 * we are the only setter, thus concurrent access is undefined
92 if (level
== KVM_INTERRUPT_SET_LEVEL
)
94 else if (level
== KVM_INTERRUPT_UNSET
) {
99 /* Attempt delivery */
100 icp_deliver_irq(xics
, NULL
, irq
);
102 return state
->asserted
;
105 static void ics_check_resend(struct kvmppc_xics
*xics
, struct kvmppc_ics
*ics
,
106 struct kvmppc_icp
*icp
)
110 mutex_lock(&ics
->lock
);
112 for (i
= 0; i
< KVMPPC_XICS_IRQ_PER_ICS
; i
++) {
113 struct ics_irq_state
*state
= &ics
->irq_state
[i
];
118 XICS_DBG("resend %#x prio %#x\n", state
->number
,
121 mutex_unlock(&ics
->lock
);
122 icp_deliver_irq(xics
, icp
, state
->number
);
123 mutex_lock(&ics
->lock
);
126 mutex_unlock(&ics
->lock
);
129 static bool write_xive(struct kvmppc_xics
*xics
, struct kvmppc_ics
*ics
,
130 struct ics_irq_state
*state
,
131 u32 server
, u32 priority
, u32 saved_priority
)
135 mutex_lock(&ics
->lock
);
137 state
->server
= server
;
138 state
->priority
= priority
;
139 state
->saved_priority
= saved_priority
;
141 if ((state
->masked_pending
|| state
->resend
) && priority
!= MASKED
) {
142 state
->masked_pending
= 0;
146 mutex_unlock(&ics
->lock
);
151 int kvmppc_xics_set_xive(struct kvm
*kvm
, u32 irq
, u32 server
, u32 priority
)
153 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
154 struct kvmppc_icp
*icp
;
155 struct kvmppc_ics
*ics
;
156 struct ics_irq_state
*state
;
162 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
165 state
= &ics
->irq_state
[src
];
167 icp
= kvmppc_xics_find_server(kvm
, server
);
171 XICS_DBG("set_xive %#x server %#x prio %#x MP:%d RS:%d\n",
172 irq
, server
, priority
,
173 state
->masked_pending
, state
->resend
);
175 if (write_xive(xics
, ics
, state
, server
, priority
, priority
))
176 icp_deliver_irq(xics
, icp
, irq
);
181 int kvmppc_xics_get_xive(struct kvm
*kvm
, u32 irq
, u32
*server
, u32
*priority
)
183 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
184 struct kvmppc_ics
*ics
;
185 struct ics_irq_state
*state
;
191 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
194 state
= &ics
->irq_state
[src
];
196 mutex_lock(&ics
->lock
);
197 *server
= state
->server
;
198 *priority
= state
->priority
;
199 mutex_unlock(&ics
->lock
);
204 int kvmppc_xics_int_on(struct kvm
*kvm
, u32 irq
)
206 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
207 struct kvmppc_icp
*icp
;
208 struct kvmppc_ics
*ics
;
209 struct ics_irq_state
*state
;
215 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
218 state
= &ics
->irq_state
[src
];
220 icp
= kvmppc_xics_find_server(kvm
, state
->server
);
224 if (write_xive(xics
, ics
, state
, state
->server
, state
->saved_priority
,
225 state
->saved_priority
))
226 icp_deliver_irq(xics
, icp
, irq
);
231 int kvmppc_xics_int_off(struct kvm
*kvm
, u32 irq
)
233 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
234 struct kvmppc_ics
*ics
;
235 struct ics_irq_state
*state
;
241 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
244 state
= &ics
->irq_state
[src
];
246 write_xive(xics
, ics
, state
, state
->server
, MASKED
, state
->priority
);
251 /* -- ICP routines, including hcalls -- */
253 static inline bool icp_try_update(struct kvmppc_icp
*icp
,
254 union kvmppc_icp_state old
,
255 union kvmppc_icp_state
new,
260 /* Calculate new output value */
261 new.out_ee
= (new.xisr
&& (new.pending_pri
< new.cppr
));
263 /* Attempt atomic update */
264 success
= cmpxchg64(&icp
->state
.raw
, old
.raw
, new.raw
) == old
.raw
;
268 XICS_DBG("UPD [%04x] - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
270 old
.cppr
, old
.mfrr
, old
.pending_pri
, old
.xisr
,
271 old
.need_resend
, old
.out_ee
);
272 XICS_DBG("UPD - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
273 new.cppr
, new.mfrr
, new.pending_pri
, new.xisr
,
274 new.need_resend
, new.out_ee
);
276 * Check for output state update
278 * Note that this is racy since another processor could be updating
279 * the state already. This is why we never clear the interrupt output
280 * here, we only ever set it. The clear only happens prior to doing
281 * an update and only by the processor itself. Currently we do it
282 * in Accept (H_XIRR) and Up_Cppr (H_XPPR).
284 * We also do not try to figure out whether the EE state has changed,
285 * we unconditionally set it if the new state calls for it. The reason
286 * for that is that we opportunistically remove the pending interrupt
287 * flag when raising CPPR, so we need to set it back here if an
288 * interrupt is still pending.
291 kvmppc_book3s_queue_irqprio(icp
->vcpu
,
292 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
294 kvmppc_fast_vcpu_kick(icp
->vcpu
);
300 static void icp_check_resend(struct kvmppc_xics
*xics
,
301 struct kvmppc_icp
*icp
)
305 /* Order this load with the test for need_resend in the caller */
307 for_each_set_bit(icsid
, icp
->resend_map
, xics
->max_icsid
+ 1) {
308 struct kvmppc_ics
*ics
= xics
->ics
[icsid
];
310 if (!test_and_clear_bit(icsid
, icp
->resend_map
))
314 ics_check_resend(xics
, ics
, icp
);
318 static bool icp_try_to_deliver(struct kvmppc_icp
*icp
, u32 irq
, u8 priority
,
321 union kvmppc_icp_state old_state
, new_state
;
324 XICS_DBG("try deliver %#x(P:%#x) to server %#x\n", irq
, priority
,
328 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
332 /* See if we can deliver */
333 success
= new_state
.cppr
> priority
&&
334 new_state
.mfrr
> priority
&&
335 new_state
.pending_pri
> priority
;
338 * If we can, check for a rejection and perform the
342 *reject
= new_state
.xisr
;
343 new_state
.xisr
= irq
;
344 new_state
.pending_pri
= priority
;
347 * If we failed to deliver we set need_resend
348 * so a subsequent CPPR state change causes us
349 * to try a new delivery.
351 new_state
.need_resend
= true;
354 } while (!icp_try_update(icp
, old_state
, new_state
, false));
359 static void icp_deliver_irq(struct kvmppc_xics
*xics
, struct kvmppc_icp
*icp
,
362 struct ics_irq_state
*state
;
363 struct kvmppc_ics
*ics
;
368 * This is used both for initial delivery of an interrupt and
369 * for subsequent rejection.
371 * Rejection can be racy vs. resends. We have evaluated the
372 * rejection in an atomic ICP transaction which is now complete,
373 * so potentially the ICP can already accept the interrupt again.
375 * So we need to retry the delivery. Essentially the reject path
376 * boils down to a failed delivery. Always.
378 * Now the interrupt could also have moved to a different target,
379 * thus we may need to re-do the ICP lookup as well
383 /* Get the ICS state and lock it */
384 ics
= kvmppc_xics_find_ics(xics
, new_irq
, &src
);
386 XICS_DBG("icp_deliver_irq: IRQ 0x%06x not found !\n", new_irq
);
389 state
= &ics
->irq_state
[src
];
391 /* Get a lock on the ICS */
392 mutex_lock(&ics
->lock
);
395 if (!icp
|| state
->server
!= icp
->server_num
) {
396 icp
= kvmppc_xics_find_server(xics
->kvm
, state
->server
);
398 pr_warn("icp_deliver_irq: IRQ 0x%06x server 0x%x not found !\n",
399 new_irq
, state
->server
);
404 /* Clear the resend bit of that interrupt */
408 * If masked, bail out
410 * Note: PAPR doesn't mention anything about masked pending
411 * when doing a resend, only when doing a delivery.
413 * However that would have the effect of losing a masked
414 * interrupt that was rejected and isn't consistent with
415 * the whole masked_pending business which is about not
416 * losing interrupts that occur while masked.
418 * I don't differenciate normal deliveries and resends, this
419 * implementation will differ from PAPR and not lose such
422 if (state
->priority
== MASKED
) {
423 XICS_DBG("irq %#x masked pending\n", new_irq
);
424 state
->masked_pending
= 1;
429 * Try the delivery, this will set the need_resend flag
430 * in the ICP as part of the atomic transaction if the
431 * delivery is not possible.
433 * Note that if successful, the new delivery might have itself
434 * rejected an interrupt that was "delivered" before we took the
437 * In this case we do the whole sequence all over again for the
438 * new guy. We cannot assume that the rejected interrupt is less
439 * favored than the new one, and thus doesn't need to be delivered,
440 * because by the time we exit icp_try_to_deliver() the target
441 * processor may well have alrady consumed & completed it, and thus
442 * the rejected interrupt might actually be already acceptable.
444 if (icp_try_to_deliver(icp
, new_irq
, state
->priority
, &reject
)) {
446 * Delivery was successful, did we reject somebody else ?
448 if (reject
&& reject
!= XICS_IPI
) {
449 mutex_unlock(&ics
->lock
);
455 * We failed to deliver the interrupt we need to set the
456 * resend map bit and mark the ICS state as needing a resend
458 set_bit(ics
->icsid
, icp
->resend_map
);
462 * If the need_resend flag got cleared in the ICP some time
463 * between icp_try_to_deliver() atomic update and now, then
464 * we know it might have missed the resend_map bit. So we
468 if (!icp
->state
.need_resend
) {
469 mutex_unlock(&ics
->lock
);
474 mutex_unlock(&ics
->lock
);
477 static void icp_down_cppr(struct kvmppc_xics
*xics
, struct kvmppc_icp
*icp
,
480 union kvmppc_icp_state old_state
, new_state
;
484 * This handles several related states in one operation:
486 * ICP State: Down_CPPR
488 * Load CPPR with new value and if the XISR is 0
489 * then check for resends:
493 * If MFRR is more favored than CPPR, check for IPIs
494 * and notify ICS of a potential resend. This is done
495 * asynchronously (when used in real mode, we will have
498 * We do not handle the complete Check_IPI as documented
499 * here. In the PAPR, this state will be used for both
500 * Set_MFRR and Down_CPPR. However, we know that we aren't
501 * changing the MFRR state here so we don't need to handle
502 * the case of an MFRR causing a reject of a pending irq,
503 * this will have been handled when the MFRR was set in the
506 * Thus we don't have to handle rejects, only resends.
508 * When implementing real mode for HV KVM, resend will lead to
509 * a H_TOO_HARD return and the whole transaction will be handled
513 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
516 new_state
.cppr
= new_cppr
;
519 * Cut down Resend / Check_IPI / IPI
521 * The logic is that we cannot have a pending interrupt
522 * trumped by an IPI at this point (see above), so we
523 * know that either the pending interrupt is already an
524 * IPI (in which case we don't care to override it) or
525 * it's either more favored than us or non existent
527 if (new_state
.mfrr
< new_cppr
&&
528 new_state
.mfrr
<= new_state
.pending_pri
) {
529 WARN_ON(new_state
.xisr
!= XICS_IPI
&&
530 new_state
.xisr
!= 0);
531 new_state
.pending_pri
= new_state
.mfrr
;
532 new_state
.xisr
= XICS_IPI
;
535 /* Latch/clear resend bit */
536 resend
= new_state
.need_resend
;
537 new_state
.need_resend
= 0;
539 } while (!icp_try_update(icp
, old_state
, new_state
, true));
542 * Now handle resend checks. Those are asynchronous to the ICP
543 * state update in HW (ie bus transactions) so we can handle them
544 * separately here too
547 icp_check_resend(xics
, icp
);
550 static noinline
unsigned long kvmppc_h_xirr(struct kvm_vcpu
*vcpu
)
552 union kvmppc_icp_state old_state
, new_state
;
553 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
556 /* First, remove EE from the processor */
557 kvmppc_book3s_dequeue_irqprio(icp
->vcpu
,
558 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
561 * ICP State: Accept_Interrupt
563 * Return the pending interrupt (if any) along with the
564 * current CPPR, then clear the XISR & set CPPR to the
568 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
570 xirr
= old_state
.xisr
| (((u32
)old_state
.cppr
) << 24);
573 new_state
.cppr
= new_state
.pending_pri
;
574 new_state
.pending_pri
= 0xff;
577 } while (!icp_try_update(icp
, old_state
, new_state
, true));
579 XICS_DBG("h_xirr vcpu %d xirr %#x\n", vcpu
->vcpu_id
, xirr
);
584 static noinline
int kvmppc_h_ipi(struct kvm_vcpu
*vcpu
, unsigned long server
,
587 union kvmppc_icp_state old_state
, new_state
;
588 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
589 struct kvmppc_icp
*icp
;
594 XICS_DBG("h_ipi vcpu %d to server %lu mfrr %#lx\n",
595 vcpu
->vcpu_id
, server
, mfrr
);
597 icp
= vcpu
->arch
.icp
;
598 local
= icp
->server_num
== server
;
600 icp
= kvmppc_xics_find_server(vcpu
->kvm
, server
);
606 * ICP state: Set_MFRR
608 * If the CPPR is more favored than the new MFRR, then
609 * nothing needs to be rejected as there can be no XISR to
610 * reject. If the MFRR is being made less favored then
611 * there might be a previously-rejected interrupt needing
614 * If the CPPR is less favored, then we might be replacing
615 * an interrupt, and thus need to possibly reject it as in
617 * ICP state: Check_IPI
620 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
623 new_state
.mfrr
= mfrr
;
628 if (mfrr
< new_state
.cppr
) {
629 /* Reject a pending interrupt if not an IPI */
630 if (mfrr
<= new_state
.pending_pri
)
631 reject
= new_state
.xisr
;
632 new_state
.pending_pri
= mfrr
;
633 new_state
.xisr
= XICS_IPI
;
636 if (mfrr
> old_state
.mfrr
&& mfrr
> new_state
.cppr
) {
637 resend
= new_state
.need_resend
;
638 new_state
.need_resend
= 0;
640 } while (!icp_try_update(icp
, old_state
, new_state
, local
));
643 if (reject
&& reject
!= XICS_IPI
)
644 icp_deliver_irq(xics
, icp
, reject
);
648 icp_check_resend(xics
, icp
);
653 static noinline
void kvmppc_h_cppr(struct kvm_vcpu
*vcpu
, unsigned long cppr
)
655 union kvmppc_icp_state old_state
, new_state
;
656 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
657 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
660 XICS_DBG("h_cppr vcpu %d cppr %#lx\n", vcpu
->vcpu_id
, cppr
);
663 * ICP State: Set_CPPR
665 * We can safely compare the new value with the current
666 * value outside of the transaction as the CPPR is only
667 * ever changed by the processor on itself
669 if (cppr
> icp
->state
.cppr
)
670 icp_down_cppr(xics
, icp
, cppr
);
671 else if (cppr
== icp
->state
.cppr
)
677 * The processor is raising its priority, this can result
678 * in a rejection of a pending interrupt:
680 * ICP State: Reject_Current
682 * We can remove EE from the current processor, the update
683 * transaction will set it again if needed
685 kvmppc_book3s_dequeue_irqprio(icp
->vcpu
,
686 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
689 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
692 new_state
.cppr
= cppr
;
694 if (cppr
<= new_state
.pending_pri
) {
695 reject
= new_state
.xisr
;
697 new_state
.pending_pri
= 0xff;
700 } while (!icp_try_update(icp
, old_state
, new_state
, true));
703 * Check for rejects. They are handled by doing a new delivery
704 * attempt (see comments in icp_deliver_irq).
706 if (reject
&& reject
!= XICS_IPI
)
707 icp_deliver_irq(xics
, icp
, reject
);
710 static noinline
int kvmppc_h_eoi(struct kvm_vcpu
*vcpu
, unsigned long xirr
)
712 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
713 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
714 struct kvmppc_ics
*ics
;
715 struct ics_irq_state
*state
;
716 u32 irq
= xirr
& 0x00ffffff;
719 XICS_DBG("h_eoi vcpu %d eoi %#lx\n", vcpu
->vcpu_id
, xirr
);
724 * Note: If EOI is incorrectly used by SW to lower the CPPR
725 * value (ie more favored), we do not check for rejection of
726 * a pending interrupt, this is a SW error and PAPR sepcifies
727 * that we don't have to deal with it.
729 * The sending of an EOI to the ICS is handled after the
732 * ICP State: Down_CPPR which we handle
733 * in a separate function as it's shared with H_CPPR.
735 icp_down_cppr(xics
, icp
, xirr
>> 24);
737 /* IPIs have no EOI */
741 * EOI handling: If the interrupt is still asserted, we need to
742 * resend it. We can take a lockless "peek" at the ICS state here.
744 * "Message" interrupts will never have "asserted" set
746 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
748 XICS_DBG("h_eoi: IRQ 0x%06x not found !\n", irq
);
751 state
= &ics
->irq_state
[src
];
753 /* Still asserted, resend it */
755 icp_deliver_irq(xics
, icp
, irq
);
760 static noinline
int kvmppc_xics_rm_complete(struct kvm_vcpu
*vcpu
, u32 hcall
)
762 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
763 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
765 XICS_DBG("XICS_RM: H_%x completing, act: %x state: %lx tgt: %p\n",
766 hcall
, icp
->rm_action
, icp
->rm_dbgstate
.raw
, icp
->rm_dbgtgt
);
768 if (icp
->rm_action
& XICS_RM_KICK_VCPU
)
769 kvmppc_fast_vcpu_kick(icp
->rm_kick_target
);
770 if (icp
->rm_action
& XICS_RM_CHECK_RESEND
)
771 icp_check_resend(xics
, icp
);
772 if (icp
->rm_action
& XICS_RM_REJECT
)
773 icp_deliver_irq(xics
, icp
, icp
->rm_reject
);
780 int kvmppc_xics_hcall(struct kvm_vcpu
*vcpu
, u32 req
)
782 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
786 /* Check if we have an ICP */
787 if (!xics
|| !vcpu
->arch
.icp
)
790 /* Check for real mode returning too hard */
792 return kvmppc_xics_rm_complete(vcpu
, req
);
796 res
= kvmppc_h_xirr(vcpu
);
797 kvmppc_set_gpr(vcpu
, 4, res
);
800 kvmppc_h_cppr(vcpu
, kvmppc_get_gpr(vcpu
, 4));
803 rc
= kvmppc_h_eoi(vcpu
, kvmppc_get_gpr(vcpu
, 4));
806 rc
= kvmppc_h_ipi(vcpu
, kvmppc_get_gpr(vcpu
, 4),
807 kvmppc_get_gpr(vcpu
, 5));
815 /* -- Initialisation code etc. -- */
817 static int xics_debug_show(struct seq_file
*m
, void *private)
819 struct kvmppc_xics
*xics
= m
->private;
820 struct kvm
*kvm
= xics
->kvm
;
821 struct kvm_vcpu
*vcpu
;
827 seq_printf(m
, "=========\nICP state\n=========\n");
829 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
830 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
831 union kvmppc_icp_state state
;
836 state
.raw
= ACCESS_ONCE(icp
->state
.raw
);
837 seq_printf(m
, "cpu server %#lx XIRR:%#x PPRI:%#x CPPR:%#x MFRR:%#x OUT:%d NR:%d\n",
838 icp
->server_num
, state
.xisr
,
839 state
.pending_pri
, state
.cppr
, state
.mfrr
,
840 state
.out_ee
, state
.need_resend
);
843 for (icsid
= 0; icsid
<= KVMPPC_XICS_MAX_ICS_ID
; icsid
++) {
844 struct kvmppc_ics
*ics
= xics
->ics
[icsid
];
849 seq_printf(m
, "=========\nICS state for ICS 0x%x\n=========\n",
852 mutex_lock(&ics
->lock
);
854 for (i
= 0; i
< KVMPPC_XICS_IRQ_PER_ICS
; i
++) {
855 struct ics_irq_state
*irq
= &ics
->irq_state
[i
];
857 seq_printf(m
, "irq 0x%06x: server %#x prio %#x save prio %#x asserted %d resend %d masked pending %d\n",
858 irq
->number
, irq
->server
, irq
->priority
,
859 irq
->saved_priority
, irq
->asserted
,
860 irq
->resend
, irq
->masked_pending
);
863 mutex_unlock(&ics
->lock
);
868 static int xics_debug_open(struct inode
*inode
, struct file
*file
)
870 return single_open(file
, xics_debug_show
, inode
->i_private
);
873 static const struct file_operations xics_debug_fops
= {
874 .open
= xics_debug_open
,
877 .release
= single_release
,
880 static void xics_debugfs_init(struct kvmppc_xics
*xics
)
884 name
= kasprintf(GFP_KERNEL
, "kvm-xics-%p", xics
);
886 pr_err("%s: no memory for name\n", __func__
);
890 xics
->dentry
= debugfs_create_file(name
, S_IRUGO
, powerpc_debugfs_root
,
891 xics
, &xics_debug_fops
);
893 pr_debug("%s: created %s\n", __func__
, name
);
897 static struct kvmppc_ics
*kvmppc_xics_create_ics(struct kvm
*kvm
,
898 struct kvmppc_xics
*xics
, int irq
)
900 struct kvmppc_ics
*ics
;
903 icsid
= irq
>> KVMPPC_XICS_ICS_SHIFT
;
905 mutex_lock(&kvm
->lock
);
907 /* ICS already exists - somebody else got here first */
908 if (xics
->ics
[icsid
])
912 ics
= kzalloc(sizeof(struct kvmppc_ics
), GFP_KERNEL
);
916 mutex_init(&ics
->lock
);
919 for (i
= 0; i
< KVMPPC_XICS_IRQ_PER_ICS
; i
++) {
920 ics
->irq_state
[i
].number
= (icsid
<< KVMPPC_XICS_ICS_SHIFT
) | i
;
921 ics
->irq_state
[i
].priority
= MASKED
;
922 ics
->irq_state
[i
].saved_priority
= MASKED
;
925 xics
->ics
[icsid
] = ics
;
927 if (icsid
> xics
->max_icsid
)
928 xics
->max_icsid
= icsid
;
931 mutex_unlock(&kvm
->lock
);
932 return xics
->ics
[icsid
];
935 int kvmppc_xics_create_icp(struct kvm_vcpu
*vcpu
, unsigned long server_num
)
937 struct kvmppc_icp
*icp
;
939 if (!vcpu
->kvm
->arch
.xics
)
942 if (kvmppc_xics_find_server(vcpu
->kvm
, server_num
))
945 icp
= kzalloc(sizeof(struct kvmppc_icp
), GFP_KERNEL
);
950 icp
->server_num
= server_num
;
951 icp
->state
.mfrr
= MASKED
;
952 icp
->state
.pending_pri
= MASKED
;
953 vcpu
->arch
.icp
= icp
;
955 XICS_DBG("created server for vcpu %d\n", vcpu
->vcpu_id
);
960 u64
kvmppc_xics_get_icp(struct kvm_vcpu
*vcpu
)
962 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
963 union kvmppc_icp_state state
;
968 return ((u64
)state
.cppr
<< KVM_REG_PPC_ICP_CPPR_SHIFT
) |
969 ((u64
)state
.xisr
<< KVM_REG_PPC_ICP_XISR_SHIFT
) |
970 ((u64
)state
.mfrr
<< KVM_REG_PPC_ICP_MFRR_SHIFT
) |
971 ((u64
)state
.pending_pri
<< KVM_REG_PPC_ICP_PPRI_SHIFT
);
974 int kvmppc_xics_set_icp(struct kvm_vcpu
*vcpu
, u64 icpval
)
976 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
977 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
978 union kvmppc_icp_state old_state
, new_state
;
979 struct kvmppc_ics
*ics
;
980 u8 cppr
, mfrr
, pending_pri
;
988 cppr
= icpval
>> KVM_REG_PPC_ICP_CPPR_SHIFT
;
989 xisr
= (icpval
>> KVM_REG_PPC_ICP_XISR_SHIFT
) &
990 KVM_REG_PPC_ICP_XISR_MASK
;
991 mfrr
= icpval
>> KVM_REG_PPC_ICP_MFRR_SHIFT
;
992 pending_pri
= icpval
>> KVM_REG_PPC_ICP_PPRI_SHIFT
;
994 /* Require the new state to be internally consistent */
996 if (pending_pri
!= 0xff)
998 } else if (xisr
== XICS_IPI
) {
999 if (pending_pri
!= mfrr
|| pending_pri
>= cppr
)
1002 if (pending_pri
>= mfrr
|| pending_pri
>= cppr
)
1004 ics
= kvmppc_xics_find_ics(xics
, xisr
, &src
);
1010 new_state
.cppr
= cppr
;
1011 new_state
.xisr
= xisr
;
1012 new_state
.mfrr
= mfrr
;
1013 new_state
.pending_pri
= pending_pri
;
1016 * Deassert the CPU interrupt request.
1017 * icp_try_update will reassert it if necessary.
1019 kvmppc_book3s_dequeue_irqprio(icp
->vcpu
,
1020 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
1023 * Note that if we displace an interrupt from old_state.xisr,
1024 * we don't mark it as rejected. We expect userspace to set
1025 * the state of the interrupt sources to be consistent with
1026 * the ICP states (either before or afterwards, which doesn't
1027 * matter). We do handle resends due to CPPR becoming less
1028 * favoured because that is necessary to end up with a
1029 * consistent state in the situation where userspace restores
1030 * the ICS states before the ICP states.
1033 old_state
= ACCESS_ONCE(icp
->state
);
1035 if (new_state
.mfrr
<= old_state
.mfrr
) {
1037 new_state
.need_resend
= old_state
.need_resend
;
1039 resend
= old_state
.need_resend
;
1040 new_state
.need_resend
= 0;
1042 } while (!icp_try_update(icp
, old_state
, new_state
, false));
1045 icp_check_resend(xics
, icp
);
1050 static int xics_get_source(struct kvmppc_xics
*xics
, long irq
, u64 addr
)
1053 struct kvmppc_ics
*ics
;
1054 struct ics_irq_state
*irqp
;
1055 u64 __user
*ubufp
= (u64 __user
*) addr
;
1059 ics
= kvmppc_xics_find_ics(xics
, irq
, &idx
);
1063 irqp
= &ics
->irq_state
[idx
];
1064 mutex_lock(&ics
->lock
);
1068 prio
= irqp
->priority
;
1069 if (prio
== MASKED
) {
1070 val
|= KVM_XICS_MASKED
;
1071 prio
= irqp
->saved_priority
;
1073 val
|= prio
<< KVM_XICS_PRIORITY_SHIFT
;
1075 val
|= KVM_XICS_LEVEL_SENSITIVE
| KVM_XICS_PENDING
;
1076 else if (irqp
->masked_pending
|| irqp
->resend
)
1077 val
|= KVM_XICS_PENDING
;
1080 mutex_unlock(&ics
->lock
);
1082 if (!ret
&& put_user(val
, ubufp
))
1088 static int xics_set_source(struct kvmppc_xics
*xics
, long irq
, u64 addr
)
1090 struct kvmppc_ics
*ics
;
1091 struct ics_irq_state
*irqp
;
1092 u64 __user
*ubufp
= (u64 __user
*) addr
;
1098 if (irq
< KVMPPC_XICS_FIRST_IRQ
|| irq
>= KVMPPC_XICS_NR_IRQS
)
1101 ics
= kvmppc_xics_find_ics(xics
, irq
, &idx
);
1103 ics
= kvmppc_xics_create_ics(xics
->kvm
, xics
, irq
);
1107 irqp
= &ics
->irq_state
[idx
];
1108 if (get_user(val
, ubufp
))
1111 server
= val
& KVM_XICS_DESTINATION_MASK
;
1112 prio
= val
>> KVM_XICS_PRIORITY_SHIFT
;
1113 if (prio
!= MASKED
&&
1114 kvmppc_xics_find_server(xics
->kvm
, server
) == NULL
)
1117 mutex_lock(&ics
->lock
);
1118 irqp
->server
= server
;
1119 irqp
->saved_priority
= prio
;
1120 if (val
& KVM_XICS_MASKED
)
1122 irqp
->priority
= prio
;
1124 irqp
->masked_pending
= 0;
1126 if ((val
& KVM_XICS_PENDING
) && (val
& KVM_XICS_LEVEL_SENSITIVE
))
1129 mutex_unlock(&ics
->lock
);
1131 if (val
& KVM_XICS_PENDING
)
1132 icp_deliver_irq(xics
, NULL
, irqp
->number
);
1137 int kvm_set_irq(struct kvm
*kvm
, int irq_source_id
, u32 irq
, int level
,
1140 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
1142 return ics_deliver_irq(xics
, irq
, level
, line_status
);
1145 static int xics_set_attr(struct kvm_device
*dev
, struct kvm_device_attr
*attr
)
1147 struct kvmppc_xics
*xics
= dev
->private;
1149 switch (attr
->group
) {
1150 case KVM_DEV_XICS_GRP_SOURCES
:
1151 return xics_set_source(xics
, attr
->attr
, attr
->addr
);
1156 static int xics_get_attr(struct kvm_device
*dev
, struct kvm_device_attr
*attr
)
1158 struct kvmppc_xics
*xics
= dev
->private;
1160 switch (attr
->group
) {
1161 case KVM_DEV_XICS_GRP_SOURCES
:
1162 return xics_get_source(xics
, attr
->attr
, attr
->addr
);
1167 static int xics_has_attr(struct kvm_device
*dev
, struct kvm_device_attr
*attr
)
1169 switch (attr
->group
) {
1170 case KVM_DEV_XICS_GRP_SOURCES
:
1171 if (attr
->attr
>= KVMPPC_XICS_FIRST_IRQ
&&
1172 attr
->attr
< KVMPPC_XICS_NR_IRQS
)
1179 static void kvmppc_xics_free(struct kvm_device
*dev
)
1181 struct kvmppc_xics
*xics
= dev
->private;
1183 struct kvm
*kvm
= xics
->kvm
;
1185 debugfs_remove(xics
->dentry
);
1188 kvm
->arch
.xics
= NULL
;
1190 for (i
= 0; i
<= xics
->max_icsid
; i
++)
1191 kfree(xics
->ics
[i
]);
1196 static int kvmppc_xics_create(struct kvm_device
*dev
, u32 type
)
1198 struct kvmppc_xics
*xics
;
1199 struct kvm
*kvm
= dev
->kvm
;
1202 xics
= kzalloc(sizeof(*xics
), GFP_KERNEL
);
1206 dev
->private = xics
;
1210 /* Already there ? */
1211 mutex_lock(&kvm
->lock
);
1215 kvm
->arch
.xics
= xics
;
1216 mutex_unlock(&kvm
->lock
);
1221 xics_debugfs_init(xics
);
1223 #ifdef CONFIG_KVM_BOOK3S_64_HV
1224 if (cpu_has_feature(CPU_FTR_ARCH_206
)) {
1225 /* Enable real mode support */
1226 xics
->real_mode
= ENABLE_REALMODE
;
1227 xics
->real_mode_dbg
= DEBUG_REALMODE
;
1229 #endif /* CONFIG_KVM_BOOK3S_64_HV */
1234 struct kvm_device_ops kvm_xics_ops
= {
1236 .create
= kvmppc_xics_create
,
1237 .destroy
= kvmppc_xics_free
,
1238 .set_attr
= xics_set_attr
,
1239 .get_attr
= xics_get_attr
,
1240 .has_attr
= xics_has_attr
,
1243 int kvmppc_xics_connect_vcpu(struct kvm_device
*dev
, struct kvm_vcpu
*vcpu
,
1246 struct kvmppc_xics
*xics
= dev
->private;
1249 if (dev
->ops
!= &kvm_xics_ops
)
1251 if (xics
->kvm
!= vcpu
->kvm
)
1253 if (vcpu
->arch
.irq_type
)
1256 r
= kvmppc_xics_create_icp(vcpu
, xcpu
);
1258 vcpu
->arch
.irq_type
= KVMPPC_IRQ_XICS
;
1263 void kvmppc_xics_free_icp(struct kvm_vcpu
*vcpu
)
1265 if (!vcpu
->arch
.icp
)
1267 kfree(vcpu
->arch
.icp
);
1268 vcpu
->arch
.icp
= NULL
;
1269 vcpu
->arch
.irq_type
= KVMPPC_IRQ_DEFAULT
;