2 * Copyright 2012 Michael Ellerman, IBM Corporation.
3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
10 #include <linux/kernel.h>
11 #include <linux/kvm_host.h>
12 #include <linux/err.h>
13 #include <linux/gfp.h>
14 #include <linux/anon_inodes.h>
16 #include <asm/uaccess.h>
17 #include <asm/kvm_book3s.h>
18 #include <asm/kvm_ppc.h>
19 #include <asm/hvcall.h>
21 #include <asm/debug.h>
23 #include <linux/debugfs.h>
24 #include <linux/seq_file.h>
26 #include "book3s_xics.h"
29 #define XICS_DBG(fmt...) do { } while (0)
31 #define XICS_DBG(fmt...) trace_printk(fmt)
34 #define ENABLE_REALMODE true
35 #define DEBUG_REALMODE false
41 * Each ICS has a mutex protecting the information about the IRQ
42 * sources and avoiding simultaneous deliveries if the same interrupt.
44 * ICP operations are done via a single compare & swap transaction
45 * (most ICP state fits in the union kvmppc_icp_state)
52 * - To speed up resends, keep a bitmap of "resend" set bits in the
55 * - Speed up server# -> ICP lookup (array ? hash table ?)
57 * - Make ICS lockless as well, or at least a per-interrupt lock or hashed
58 * locks array to improve scalability
61 /* -- ICS routines -- */
63 static void icp_deliver_irq(struct kvmppc_xics
*xics
, struct kvmppc_icp
*icp
,
66 static int ics_deliver_irq(struct kvmppc_xics
*xics
, u32 irq
, u32 level
,
69 struct ics_irq_state
*state
;
70 struct kvmppc_ics
*ics
;
73 XICS_DBG("ics deliver %#x (level: %d)\n", irq
, level
);
75 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
77 XICS_DBG("ics_deliver_irq: IRQ 0x%06x not found !\n", irq
);
80 state
= &ics
->irq_state
[src
];
85 return state
->asserted
;
88 * We set state->asserted locklessly. This should be fine as
89 * we are the only setter, thus concurrent access is undefined
92 if (level
== KVM_INTERRUPT_SET_LEVEL
)
94 else if (level
== KVM_INTERRUPT_UNSET
) {
99 /* Attempt delivery */
100 icp_deliver_irq(xics
, NULL
, irq
);
102 return state
->asserted
;
105 static void ics_check_resend(struct kvmppc_xics
*xics
, struct kvmppc_ics
*ics
,
106 struct kvmppc_icp
*icp
)
110 mutex_lock(&ics
->lock
);
112 for (i
= 0; i
< KVMPPC_XICS_IRQ_PER_ICS
; i
++) {
113 struct ics_irq_state
*state
= &ics
->irq_state
[i
];
118 XICS_DBG("resend %#x prio %#x\n", state
->number
,
121 mutex_unlock(&ics
->lock
);
122 icp_deliver_irq(xics
, icp
, state
->number
);
123 mutex_lock(&ics
->lock
);
126 mutex_unlock(&ics
->lock
);
129 static bool write_xive(struct kvmppc_xics
*xics
, struct kvmppc_ics
*ics
,
130 struct ics_irq_state
*state
,
131 u32 server
, u32 priority
, u32 saved_priority
)
135 mutex_lock(&ics
->lock
);
137 state
->server
= server
;
138 state
->priority
= priority
;
139 state
->saved_priority
= saved_priority
;
141 if ((state
->masked_pending
|| state
->resend
) && priority
!= MASKED
) {
142 state
->masked_pending
= 0;
146 mutex_unlock(&ics
->lock
);
151 int kvmppc_xics_set_xive(struct kvm
*kvm
, u32 irq
, u32 server
, u32 priority
)
153 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
154 struct kvmppc_icp
*icp
;
155 struct kvmppc_ics
*ics
;
156 struct ics_irq_state
*state
;
162 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
165 state
= &ics
->irq_state
[src
];
167 icp
= kvmppc_xics_find_server(kvm
, server
);
171 XICS_DBG("set_xive %#x server %#x prio %#x MP:%d RS:%d\n",
172 irq
, server
, priority
,
173 state
->masked_pending
, state
->resend
);
175 if (write_xive(xics
, ics
, state
, server
, priority
, priority
))
176 icp_deliver_irq(xics
, icp
, irq
);
181 int kvmppc_xics_get_xive(struct kvm
*kvm
, u32 irq
, u32
*server
, u32
*priority
)
183 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
184 struct kvmppc_ics
*ics
;
185 struct ics_irq_state
*state
;
191 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
194 state
= &ics
->irq_state
[src
];
196 mutex_lock(&ics
->lock
);
197 *server
= state
->server
;
198 *priority
= state
->priority
;
199 mutex_unlock(&ics
->lock
);
204 int kvmppc_xics_int_on(struct kvm
*kvm
, u32 irq
)
206 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
207 struct kvmppc_icp
*icp
;
208 struct kvmppc_ics
*ics
;
209 struct ics_irq_state
*state
;
215 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
218 state
= &ics
->irq_state
[src
];
220 icp
= kvmppc_xics_find_server(kvm
, state
->server
);
224 if (write_xive(xics
, ics
, state
, state
->server
, state
->saved_priority
,
225 state
->saved_priority
))
226 icp_deliver_irq(xics
, icp
, irq
);
231 int kvmppc_xics_int_off(struct kvm
*kvm
, u32 irq
)
233 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
234 struct kvmppc_ics
*ics
;
235 struct ics_irq_state
*state
;
241 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
244 state
= &ics
->irq_state
[src
];
246 write_xive(xics
, ics
, state
, state
->server
, MASKED
, state
->priority
);
251 /* -- ICP routines, including hcalls -- */
253 static inline bool icp_try_update(struct kvmppc_icp
*icp
,
254 union kvmppc_icp_state old
,
255 union kvmppc_icp_state
new,
260 /* Calculate new output value */
261 new.out_ee
= (new.xisr
&& (new.pending_pri
< new.cppr
));
263 /* Attempt atomic update */
264 success
= cmpxchg64(&icp
->state
.raw
, old
.raw
, new.raw
) == old
.raw
;
268 XICS_DBG("UPD [%04x] - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
270 old
.cppr
, old
.mfrr
, old
.pending_pri
, old
.xisr
,
271 old
.need_resend
, old
.out_ee
);
272 XICS_DBG("UPD - C:%02x M:%02x PP: %02x PI:%06x R:%d O:%d\n",
273 new.cppr
, new.mfrr
, new.pending_pri
, new.xisr
,
274 new.need_resend
, new.out_ee
);
276 * Check for output state update
278 * Note that this is racy since another processor could be updating
279 * the state already. This is why we never clear the interrupt output
280 * here, we only ever set it. The clear only happens prior to doing
281 * an update and only by the processor itself. Currently we do it
282 * in Accept (H_XIRR) and Up_Cppr (H_XPPR).
284 * We also do not try to figure out whether the EE state has changed,
285 * we unconditionally set it if the new state calls for it. The reason
286 * for that is that we opportunistically remove the pending interrupt
287 * flag when raising CPPR, so we need to set it back here if an
288 * interrupt is still pending.
291 kvmppc_book3s_queue_irqprio(icp
->vcpu
,
292 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
294 kvmppc_fast_vcpu_kick(icp
->vcpu
);
300 static void icp_check_resend(struct kvmppc_xics
*xics
,
301 struct kvmppc_icp
*icp
)
305 /* Order this load with the test for need_resend in the caller */
307 for_each_set_bit(icsid
, icp
->resend_map
, xics
->max_icsid
+ 1) {
308 struct kvmppc_ics
*ics
= xics
->ics
[icsid
];
310 if (!test_and_clear_bit(icsid
, icp
->resend_map
))
314 ics_check_resend(xics
, ics
, icp
);
318 static bool icp_try_to_deliver(struct kvmppc_icp
*icp
, u32 irq
, u8 priority
,
321 union kvmppc_icp_state old_state
, new_state
;
324 XICS_DBG("try deliver %#x(P:%#x) to server %#x\n", irq
, priority
,
328 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
332 /* See if we can deliver */
333 success
= new_state
.cppr
> priority
&&
334 new_state
.mfrr
> priority
&&
335 new_state
.pending_pri
> priority
;
338 * If we can, check for a rejection and perform the
342 *reject
= new_state
.xisr
;
343 new_state
.xisr
= irq
;
344 new_state
.pending_pri
= priority
;
347 * If we failed to deliver we set need_resend
348 * so a subsequent CPPR state change causes us
349 * to try a new delivery.
351 new_state
.need_resend
= true;
354 } while (!icp_try_update(icp
, old_state
, new_state
, false));
359 static void icp_deliver_irq(struct kvmppc_xics
*xics
, struct kvmppc_icp
*icp
,
362 struct ics_irq_state
*state
;
363 struct kvmppc_ics
*ics
;
368 * This is used both for initial delivery of an interrupt and
369 * for subsequent rejection.
371 * Rejection can be racy vs. resends. We have evaluated the
372 * rejection in an atomic ICP transaction which is now complete,
373 * so potentially the ICP can already accept the interrupt again.
375 * So we need to retry the delivery. Essentially the reject path
376 * boils down to a failed delivery. Always.
378 * Now the interrupt could also have moved to a different target,
379 * thus we may need to re-do the ICP lookup as well
383 /* Get the ICS state and lock it */
384 ics
= kvmppc_xics_find_ics(xics
, new_irq
, &src
);
386 XICS_DBG("icp_deliver_irq: IRQ 0x%06x not found !\n", new_irq
);
389 state
= &ics
->irq_state
[src
];
391 /* Get a lock on the ICS */
392 mutex_lock(&ics
->lock
);
395 if (!icp
|| state
->server
!= icp
->server_num
) {
396 icp
= kvmppc_xics_find_server(xics
->kvm
, state
->server
);
398 pr_warn("icp_deliver_irq: IRQ 0x%06x server 0x%x not found !\n",
399 new_irq
, state
->server
);
404 /* Clear the resend bit of that interrupt */
408 * If masked, bail out
410 * Note: PAPR doesn't mention anything about masked pending
411 * when doing a resend, only when doing a delivery.
413 * However that would have the effect of losing a masked
414 * interrupt that was rejected and isn't consistent with
415 * the whole masked_pending business which is about not
416 * losing interrupts that occur while masked.
418 * I don't differenciate normal deliveries and resends, this
419 * implementation will differ from PAPR and not lose such
422 if (state
->priority
== MASKED
) {
423 XICS_DBG("irq %#x masked pending\n", new_irq
);
424 state
->masked_pending
= 1;
429 * Try the delivery, this will set the need_resend flag
430 * in the ICP as part of the atomic transaction if the
431 * delivery is not possible.
433 * Note that if successful, the new delivery might have itself
434 * rejected an interrupt that was "delivered" before we took the
437 * In this case we do the whole sequence all over again for the
438 * new guy. We cannot assume that the rejected interrupt is less
439 * favored than the new one, and thus doesn't need to be delivered,
440 * because by the time we exit icp_try_to_deliver() the target
441 * processor may well have alrady consumed & completed it, and thus
442 * the rejected interrupt might actually be already acceptable.
444 if (icp_try_to_deliver(icp
, new_irq
, state
->priority
, &reject
)) {
446 * Delivery was successful, did we reject somebody else ?
448 if (reject
&& reject
!= XICS_IPI
) {
449 mutex_unlock(&ics
->lock
);
455 * We failed to deliver the interrupt we need to set the
456 * resend map bit and mark the ICS state as needing a resend
458 set_bit(ics
->icsid
, icp
->resend_map
);
462 * If the need_resend flag got cleared in the ICP some time
463 * between icp_try_to_deliver() atomic update and now, then
464 * we know it might have missed the resend_map bit. So we
468 if (!icp
->state
.need_resend
) {
469 mutex_unlock(&ics
->lock
);
474 mutex_unlock(&ics
->lock
);
477 static void icp_down_cppr(struct kvmppc_xics
*xics
, struct kvmppc_icp
*icp
,
480 union kvmppc_icp_state old_state
, new_state
;
484 * This handles several related states in one operation:
486 * ICP State: Down_CPPR
488 * Load CPPR with new value and if the XISR is 0
489 * then check for resends:
493 * If MFRR is more favored than CPPR, check for IPIs
494 * and notify ICS of a potential resend. This is done
495 * asynchronously (when used in real mode, we will have
498 * We do not handle the complete Check_IPI as documented
499 * here. In the PAPR, this state will be used for both
500 * Set_MFRR and Down_CPPR. However, we know that we aren't
501 * changing the MFRR state here so we don't need to handle
502 * the case of an MFRR causing a reject of a pending irq,
503 * this will have been handled when the MFRR was set in the
506 * Thus we don't have to handle rejects, only resends.
508 * When implementing real mode for HV KVM, resend will lead to
509 * a H_TOO_HARD return and the whole transaction will be handled
513 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
516 new_state
.cppr
= new_cppr
;
519 * Cut down Resend / Check_IPI / IPI
521 * The logic is that we cannot have a pending interrupt
522 * trumped by an IPI at this point (see above), so we
523 * know that either the pending interrupt is already an
524 * IPI (in which case we don't care to override it) or
525 * it's either more favored than us or non existent
527 if (new_state
.mfrr
< new_cppr
&&
528 new_state
.mfrr
<= new_state
.pending_pri
) {
529 WARN_ON(new_state
.xisr
!= XICS_IPI
&&
530 new_state
.xisr
!= 0);
531 new_state
.pending_pri
= new_state
.mfrr
;
532 new_state
.xisr
= XICS_IPI
;
535 /* Latch/clear resend bit */
536 resend
= new_state
.need_resend
;
537 new_state
.need_resend
= 0;
539 } while (!icp_try_update(icp
, old_state
, new_state
, true));
542 * Now handle resend checks. Those are asynchronous to the ICP
543 * state update in HW (ie bus transactions) so we can handle them
544 * separately here too
547 icp_check_resend(xics
, icp
);
550 static noinline
unsigned long kvmppc_h_xirr(struct kvm_vcpu
*vcpu
)
552 union kvmppc_icp_state old_state
, new_state
;
553 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
556 /* First, remove EE from the processor */
557 kvmppc_book3s_dequeue_irqprio(icp
->vcpu
,
558 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
561 * ICP State: Accept_Interrupt
563 * Return the pending interrupt (if any) along with the
564 * current CPPR, then clear the XISR & set CPPR to the
568 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
570 xirr
= old_state
.xisr
| (((u32
)old_state
.cppr
) << 24);
573 new_state
.cppr
= new_state
.pending_pri
;
574 new_state
.pending_pri
= 0xff;
577 } while (!icp_try_update(icp
, old_state
, new_state
, true));
579 XICS_DBG("h_xirr vcpu %d xirr %#x\n", vcpu
->vcpu_id
, xirr
);
584 static noinline
int kvmppc_h_ipi(struct kvm_vcpu
*vcpu
, unsigned long server
,
587 union kvmppc_icp_state old_state
, new_state
;
588 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
589 struct kvmppc_icp
*icp
;
594 XICS_DBG("h_ipi vcpu %d to server %lu mfrr %#lx\n",
595 vcpu
->vcpu_id
, server
, mfrr
);
597 icp
= vcpu
->arch
.icp
;
598 local
= icp
->server_num
== server
;
600 icp
= kvmppc_xics_find_server(vcpu
->kvm
, server
);
606 * ICP state: Set_MFRR
608 * If the CPPR is more favored than the new MFRR, then
609 * nothing needs to be rejected as there can be no XISR to
610 * reject. If the MFRR is being made less favored then
611 * there might be a previously-rejected interrupt needing
614 * If the CPPR is less favored, then we might be replacing
615 * an interrupt, and thus need to possibly reject it as in
617 * ICP state: Check_IPI
620 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
623 new_state
.mfrr
= mfrr
;
628 if (mfrr
< new_state
.cppr
) {
629 /* Reject a pending interrupt if not an IPI */
630 if (mfrr
<= new_state
.pending_pri
)
631 reject
= new_state
.xisr
;
632 new_state
.pending_pri
= mfrr
;
633 new_state
.xisr
= XICS_IPI
;
636 if (mfrr
> old_state
.mfrr
&& mfrr
> new_state
.cppr
) {
637 resend
= new_state
.need_resend
;
638 new_state
.need_resend
= 0;
640 } while (!icp_try_update(icp
, old_state
, new_state
, local
));
643 if (reject
&& reject
!= XICS_IPI
)
644 icp_deliver_irq(xics
, icp
, reject
);
648 icp_check_resend(xics
, icp
);
653 static int kvmppc_h_ipoll(struct kvm_vcpu
*vcpu
, unsigned long server
)
655 union kvmppc_icp_state state
;
656 struct kvmppc_icp
*icp
;
658 icp
= vcpu
->arch
.icp
;
659 if (icp
->server_num
!= server
) {
660 icp
= kvmppc_xics_find_server(vcpu
->kvm
, server
);
664 state
= ACCESS_ONCE(icp
->state
);
665 kvmppc_set_gpr(vcpu
, 4, ((u32
)state
.cppr
<< 24) | state
.xisr
);
666 kvmppc_set_gpr(vcpu
, 5, state
.mfrr
);
670 static noinline
void kvmppc_h_cppr(struct kvm_vcpu
*vcpu
, unsigned long cppr
)
672 union kvmppc_icp_state old_state
, new_state
;
673 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
674 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
677 XICS_DBG("h_cppr vcpu %d cppr %#lx\n", vcpu
->vcpu_id
, cppr
);
680 * ICP State: Set_CPPR
682 * We can safely compare the new value with the current
683 * value outside of the transaction as the CPPR is only
684 * ever changed by the processor on itself
686 if (cppr
> icp
->state
.cppr
)
687 icp_down_cppr(xics
, icp
, cppr
);
688 else if (cppr
== icp
->state
.cppr
)
694 * The processor is raising its priority, this can result
695 * in a rejection of a pending interrupt:
697 * ICP State: Reject_Current
699 * We can remove EE from the current processor, the update
700 * transaction will set it again if needed
702 kvmppc_book3s_dequeue_irqprio(icp
->vcpu
,
703 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
706 old_state
= new_state
= ACCESS_ONCE(icp
->state
);
709 new_state
.cppr
= cppr
;
711 if (cppr
<= new_state
.pending_pri
) {
712 reject
= new_state
.xisr
;
714 new_state
.pending_pri
= 0xff;
717 } while (!icp_try_update(icp
, old_state
, new_state
, true));
720 * Check for rejects. They are handled by doing a new delivery
721 * attempt (see comments in icp_deliver_irq).
723 if (reject
&& reject
!= XICS_IPI
)
724 icp_deliver_irq(xics
, icp
, reject
);
727 static noinline
int kvmppc_h_eoi(struct kvm_vcpu
*vcpu
, unsigned long xirr
)
729 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
730 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
731 struct kvmppc_ics
*ics
;
732 struct ics_irq_state
*state
;
733 u32 irq
= xirr
& 0x00ffffff;
736 XICS_DBG("h_eoi vcpu %d eoi %#lx\n", vcpu
->vcpu_id
, xirr
);
741 * Note: If EOI is incorrectly used by SW to lower the CPPR
742 * value (ie more favored), we do not check for rejection of
743 * a pending interrupt, this is a SW error and PAPR sepcifies
744 * that we don't have to deal with it.
746 * The sending of an EOI to the ICS is handled after the
749 * ICP State: Down_CPPR which we handle
750 * in a separate function as it's shared with H_CPPR.
752 icp_down_cppr(xics
, icp
, xirr
>> 24);
754 /* IPIs have no EOI */
758 * EOI handling: If the interrupt is still asserted, we need to
759 * resend it. We can take a lockless "peek" at the ICS state here.
761 * "Message" interrupts will never have "asserted" set
763 ics
= kvmppc_xics_find_ics(xics
, irq
, &src
);
765 XICS_DBG("h_eoi: IRQ 0x%06x not found !\n", irq
);
768 state
= &ics
->irq_state
[src
];
770 /* Still asserted, resend it */
772 icp_deliver_irq(xics
, icp
, irq
);
777 static noinline
int kvmppc_xics_rm_complete(struct kvm_vcpu
*vcpu
, u32 hcall
)
779 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
780 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
782 XICS_DBG("XICS_RM: H_%x completing, act: %x state: %lx tgt: %p\n",
783 hcall
, icp
->rm_action
, icp
->rm_dbgstate
.raw
, icp
->rm_dbgtgt
);
785 if (icp
->rm_action
& XICS_RM_KICK_VCPU
)
786 kvmppc_fast_vcpu_kick(icp
->rm_kick_target
);
787 if (icp
->rm_action
& XICS_RM_CHECK_RESEND
)
788 icp_check_resend(xics
, icp
);
789 if (icp
->rm_action
& XICS_RM_REJECT
)
790 icp_deliver_irq(xics
, icp
, icp
->rm_reject
);
797 int kvmppc_xics_hcall(struct kvm_vcpu
*vcpu
, u32 req
)
799 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
803 /* Check if we have an ICP */
804 if (!xics
|| !vcpu
->arch
.icp
)
807 /* These requests don't have real-mode implementations at present */
810 res
= kvmppc_h_xirr(vcpu
);
811 kvmppc_set_gpr(vcpu
, 4, res
);
812 kvmppc_set_gpr(vcpu
, 5, get_tb());
815 rc
= kvmppc_h_ipoll(vcpu
, kvmppc_get_gpr(vcpu
, 4));
819 /* Check for real mode returning too hard */
821 return kvmppc_xics_rm_complete(vcpu
, req
);
825 res
= kvmppc_h_xirr(vcpu
);
826 kvmppc_set_gpr(vcpu
, 4, res
);
829 kvmppc_h_cppr(vcpu
, kvmppc_get_gpr(vcpu
, 4));
832 rc
= kvmppc_h_eoi(vcpu
, kvmppc_get_gpr(vcpu
, 4));
835 rc
= kvmppc_h_ipi(vcpu
, kvmppc_get_gpr(vcpu
, 4),
836 kvmppc_get_gpr(vcpu
, 5));
844 /* -- Initialisation code etc. -- */
846 static int xics_debug_show(struct seq_file
*m
, void *private)
848 struct kvmppc_xics
*xics
= m
->private;
849 struct kvm
*kvm
= xics
->kvm
;
850 struct kvm_vcpu
*vcpu
;
856 seq_printf(m
, "=========\nICP state\n=========\n");
858 kvm_for_each_vcpu(i
, vcpu
, kvm
) {
859 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
860 union kvmppc_icp_state state
;
865 state
.raw
= ACCESS_ONCE(icp
->state
.raw
);
866 seq_printf(m
, "cpu server %#lx XIRR:%#x PPRI:%#x CPPR:%#x MFRR:%#x OUT:%d NR:%d\n",
867 icp
->server_num
, state
.xisr
,
868 state
.pending_pri
, state
.cppr
, state
.mfrr
,
869 state
.out_ee
, state
.need_resend
);
872 for (icsid
= 0; icsid
<= KVMPPC_XICS_MAX_ICS_ID
; icsid
++) {
873 struct kvmppc_ics
*ics
= xics
->ics
[icsid
];
878 seq_printf(m
, "=========\nICS state for ICS 0x%x\n=========\n",
881 mutex_lock(&ics
->lock
);
883 for (i
= 0; i
< KVMPPC_XICS_IRQ_PER_ICS
; i
++) {
884 struct ics_irq_state
*irq
= &ics
->irq_state
[i
];
886 seq_printf(m
, "irq 0x%06x: server %#x prio %#x save prio %#x asserted %d resend %d masked pending %d\n",
887 irq
->number
, irq
->server
, irq
->priority
,
888 irq
->saved_priority
, irq
->asserted
,
889 irq
->resend
, irq
->masked_pending
);
892 mutex_unlock(&ics
->lock
);
897 static int xics_debug_open(struct inode
*inode
, struct file
*file
)
899 return single_open(file
, xics_debug_show
, inode
->i_private
);
902 static const struct file_operations xics_debug_fops
= {
903 .open
= xics_debug_open
,
906 .release
= single_release
,
909 static void xics_debugfs_init(struct kvmppc_xics
*xics
)
913 name
= kasprintf(GFP_KERNEL
, "kvm-xics-%p", xics
);
915 pr_err("%s: no memory for name\n", __func__
);
919 xics
->dentry
= debugfs_create_file(name
, S_IRUGO
, powerpc_debugfs_root
,
920 xics
, &xics_debug_fops
);
922 pr_debug("%s: created %s\n", __func__
, name
);
926 static struct kvmppc_ics
*kvmppc_xics_create_ics(struct kvm
*kvm
,
927 struct kvmppc_xics
*xics
, int irq
)
929 struct kvmppc_ics
*ics
;
932 icsid
= irq
>> KVMPPC_XICS_ICS_SHIFT
;
934 mutex_lock(&kvm
->lock
);
936 /* ICS already exists - somebody else got here first */
937 if (xics
->ics
[icsid
])
941 ics
= kzalloc(sizeof(struct kvmppc_ics
), GFP_KERNEL
);
945 mutex_init(&ics
->lock
);
948 for (i
= 0; i
< KVMPPC_XICS_IRQ_PER_ICS
; i
++) {
949 ics
->irq_state
[i
].number
= (icsid
<< KVMPPC_XICS_ICS_SHIFT
) | i
;
950 ics
->irq_state
[i
].priority
= MASKED
;
951 ics
->irq_state
[i
].saved_priority
= MASKED
;
954 xics
->ics
[icsid
] = ics
;
956 if (icsid
> xics
->max_icsid
)
957 xics
->max_icsid
= icsid
;
960 mutex_unlock(&kvm
->lock
);
961 return xics
->ics
[icsid
];
964 int kvmppc_xics_create_icp(struct kvm_vcpu
*vcpu
, unsigned long server_num
)
966 struct kvmppc_icp
*icp
;
968 if (!vcpu
->kvm
->arch
.xics
)
971 if (kvmppc_xics_find_server(vcpu
->kvm
, server_num
))
974 icp
= kzalloc(sizeof(struct kvmppc_icp
), GFP_KERNEL
);
979 icp
->server_num
= server_num
;
980 icp
->state
.mfrr
= MASKED
;
981 icp
->state
.pending_pri
= MASKED
;
982 vcpu
->arch
.icp
= icp
;
984 XICS_DBG("created server for vcpu %d\n", vcpu
->vcpu_id
);
989 u64
kvmppc_xics_get_icp(struct kvm_vcpu
*vcpu
)
991 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
992 union kvmppc_icp_state state
;
997 return ((u64
)state
.cppr
<< KVM_REG_PPC_ICP_CPPR_SHIFT
) |
998 ((u64
)state
.xisr
<< KVM_REG_PPC_ICP_XISR_SHIFT
) |
999 ((u64
)state
.mfrr
<< KVM_REG_PPC_ICP_MFRR_SHIFT
) |
1000 ((u64
)state
.pending_pri
<< KVM_REG_PPC_ICP_PPRI_SHIFT
);
1003 int kvmppc_xics_set_icp(struct kvm_vcpu
*vcpu
, u64 icpval
)
1005 struct kvmppc_icp
*icp
= vcpu
->arch
.icp
;
1006 struct kvmppc_xics
*xics
= vcpu
->kvm
->arch
.xics
;
1007 union kvmppc_icp_state old_state
, new_state
;
1008 struct kvmppc_ics
*ics
;
1009 u8 cppr
, mfrr
, pending_pri
;
1017 cppr
= icpval
>> KVM_REG_PPC_ICP_CPPR_SHIFT
;
1018 xisr
= (icpval
>> KVM_REG_PPC_ICP_XISR_SHIFT
) &
1019 KVM_REG_PPC_ICP_XISR_MASK
;
1020 mfrr
= icpval
>> KVM_REG_PPC_ICP_MFRR_SHIFT
;
1021 pending_pri
= icpval
>> KVM_REG_PPC_ICP_PPRI_SHIFT
;
1023 /* Require the new state to be internally consistent */
1025 if (pending_pri
!= 0xff)
1027 } else if (xisr
== XICS_IPI
) {
1028 if (pending_pri
!= mfrr
|| pending_pri
>= cppr
)
1031 if (pending_pri
>= mfrr
|| pending_pri
>= cppr
)
1033 ics
= kvmppc_xics_find_ics(xics
, xisr
, &src
);
1039 new_state
.cppr
= cppr
;
1040 new_state
.xisr
= xisr
;
1041 new_state
.mfrr
= mfrr
;
1042 new_state
.pending_pri
= pending_pri
;
1045 * Deassert the CPU interrupt request.
1046 * icp_try_update will reassert it if necessary.
1048 kvmppc_book3s_dequeue_irqprio(icp
->vcpu
,
1049 BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
1052 * Note that if we displace an interrupt from old_state.xisr,
1053 * we don't mark it as rejected. We expect userspace to set
1054 * the state of the interrupt sources to be consistent with
1055 * the ICP states (either before or afterwards, which doesn't
1056 * matter). We do handle resends due to CPPR becoming less
1057 * favoured because that is necessary to end up with a
1058 * consistent state in the situation where userspace restores
1059 * the ICS states before the ICP states.
1062 old_state
= ACCESS_ONCE(icp
->state
);
1064 if (new_state
.mfrr
<= old_state
.mfrr
) {
1066 new_state
.need_resend
= old_state
.need_resend
;
1068 resend
= old_state
.need_resend
;
1069 new_state
.need_resend
= 0;
1071 } while (!icp_try_update(icp
, old_state
, new_state
, false));
1074 icp_check_resend(xics
, icp
);
1079 static int xics_get_source(struct kvmppc_xics
*xics
, long irq
, u64 addr
)
1082 struct kvmppc_ics
*ics
;
1083 struct ics_irq_state
*irqp
;
1084 u64 __user
*ubufp
= (u64 __user
*) addr
;
1088 ics
= kvmppc_xics_find_ics(xics
, irq
, &idx
);
1092 irqp
= &ics
->irq_state
[idx
];
1093 mutex_lock(&ics
->lock
);
1097 prio
= irqp
->priority
;
1098 if (prio
== MASKED
) {
1099 val
|= KVM_XICS_MASKED
;
1100 prio
= irqp
->saved_priority
;
1102 val
|= prio
<< KVM_XICS_PRIORITY_SHIFT
;
1104 val
|= KVM_XICS_LEVEL_SENSITIVE
| KVM_XICS_PENDING
;
1105 else if (irqp
->masked_pending
|| irqp
->resend
)
1106 val
|= KVM_XICS_PENDING
;
1109 mutex_unlock(&ics
->lock
);
1111 if (!ret
&& put_user(val
, ubufp
))
1117 static int xics_set_source(struct kvmppc_xics
*xics
, long irq
, u64 addr
)
1119 struct kvmppc_ics
*ics
;
1120 struct ics_irq_state
*irqp
;
1121 u64 __user
*ubufp
= (u64 __user
*) addr
;
1127 if (irq
< KVMPPC_XICS_FIRST_IRQ
|| irq
>= KVMPPC_XICS_NR_IRQS
)
1130 ics
= kvmppc_xics_find_ics(xics
, irq
, &idx
);
1132 ics
= kvmppc_xics_create_ics(xics
->kvm
, xics
, irq
);
1136 irqp
= &ics
->irq_state
[idx
];
1137 if (get_user(val
, ubufp
))
1140 server
= val
& KVM_XICS_DESTINATION_MASK
;
1141 prio
= val
>> KVM_XICS_PRIORITY_SHIFT
;
1142 if (prio
!= MASKED
&&
1143 kvmppc_xics_find_server(xics
->kvm
, server
) == NULL
)
1146 mutex_lock(&ics
->lock
);
1147 irqp
->server
= server
;
1148 irqp
->saved_priority
= prio
;
1149 if (val
& KVM_XICS_MASKED
)
1151 irqp
->priority
= prio
;
1153 irqp
->masked_pending
= 0;
1155 if ((val
& KVM_XICS_PENDING
) && (val
& KVM_XICS_LEVEL_SENSITIVE
))
1158 mutex_unlock(&ics
->lock
);
1160 if (val
& KVM_XICS_PENDING
)
1161 icp_deliver_irq(xics
, NULL
, irqp
->number
);
1166 int kvm_set_irq(struct kvm
*kvm
, int irq_source_id
, u32 irq
, int level
,
1169 struct kvmppc_xics
*xics
= kvm
->arch
.xics
;
1171 return ics_deliver_irq(xics
, irq
, level
, line_status
);
1174 static int xics_set_attr(struct kvm_device
*dev
, struct kvm_device_attr
*attr
)
1176 struct kvmppc_xics
*xics
= dev
->private;
1178 switch (attr
->group
) {
1179 case KVM_DEV_XICS_GRP_SOURCES
:
1180 return xics_set_source(xics
, attr
->attr
, attr
->addr
);
1185 static int xics_get_attr(struct kvm_device
*dev
, struct kvm_device_attr
*attr
)
1187 struct kvmppc_xics
*xics
= dev
->private;
1189 switch (attr
->group
) {
1190 case KVM_DEV_XICS_GRP_SOURCES
:
1191 return xics_get_source(xics
, attr
->attr
, attr
->addr
);
1196 static int xics_has_attr(struct kvm_device
*dev
, struct kvm_device_attr
*attr
)
1198 switch (attr
->group
) {
1199 case KVM_DEV_XICS_GRP_SOURCES
:
1200 if (attr
->attr
>= KVMPPC_XICS_FIRST_IRQ
&&
1201 attr
->attr
< KVMPPC_XICS_NR_IRQS
)
1208 static void kvmppc_xics_free(struct kvm_device
*dev
)
1210 struct kvmppc_xics
*xics
= dev
->private;
1212 struct kvm
*kvm
= xics
->kvm
;
1214 debugfs_remove(xics
->dentry
);
1217 kvm
->arch
.xics
= NULL
;
1219 for (i
= 0; i
<= xics
->max_icsid
; i
++)
1220 kfree(xics
->ics
[i
]);
1225 static int kvmppc_xics_create(struct kvm_device
*dev
, u32 type
)
1227 struct kvmppc_xics
*xics
;
1228 struct kvm
*kvm
= dev
->kvm
;
1231 xics
= kzalloc(sizeof(*xics
), GFP_KERNEL
);
1235 dev
->private = xics
;
1239 /* Already there ? */
1240 mutex_lock(&kvm
->lock
);
1244 kvm
->arch
.xics
= xics
;
1245 mutex_unlock(&kvm
->lock
);
1250 xics_debugfs_init(xics
);
1252 #ifdef CONFIG_KVM_BOOK3S_64_HV
1253 if (cpu_has_feature(CPU_FTR_ARCH_206
)) {
1254 /* Enable real mode support */
1255 xics
->real_mode
= ENABLE_REALMODE
;
1256 xics
->real_mode_dbg
= DEBUG_REALMODE
;
1258 #endif /* CONFIG_KVM_BOOK3S_64_HV */
1263 struct kvm_device_ops kvm_xics_ops
= {
1265 .create
= kvmppc_xics_create
,
1266 .destroy
= kvmppc_xics_free
,
1267 .set_attr
= xics_set_attr
,
1268 .get_attr
= xics_get_attr
,
1269 .has_attr
= xics_has_attr
,
1272 int kvmppc_xics_connect_vcpu(struct kvm_device
*dev
, struct kvm_vcpu
*vcpu
,
1275 struct kvmppc_xics
*xics
= dev
->private;
1278 if (dev
->ops
!= &kvm_xics_ops
)
1280 if (xics
->kvm
!= vcpu
->kvm
)
1282 if (vcpu
->arch
.irq_type
)
1285 r
= kvmppc_xics_create_icp(vcpu
, xcpu
);
1287 vcpu
->arch
.irq_type
= KVMPPC_IRQ_XICS
;
1292 void kvmppc_xics_free_icp(struct kvm_vcpu
*vcpu
)
1294 if (!vcpu
->arch
.icp
)
1296 kfree(vcpu
->arch
.icp
);
1297 vcpu
->arch
.icp
= NULL
;
1298 vcpu
->arch
.irq_type
= KVMPPC_IRQ_DEFAULT
;