[PATCH v4 08/42] xen/riscv: distinguish h/w and virtual vINTC sources

[Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx>]

A vIRQ of a domain's vINTC is either a h/w source, one a physical IRQ is
routed to by route_irq_to_guest(), or a virtual source, whose line is
driven by the emulator of a device in Xen and whose state exists in the
vINTC only. used_irqs only tells that a vIRQ is taken, not which of the
two it is, whereas they have to be told apart: only a h/w source has a
physical IRQ to release when the domain goes away, and h/w registers to
keep in line with the guest's view.

Track the h/w sources in a new hw_irqs bitmap, the subset of used_irqs
a h/w IRQ is routed to. intc_route_irq_to_guest() sets the vIRQ's bit
there, so it now takes the domain and the vIRQ, and
domain_vintc_deinit() releases the h/w sources only.

An IRQ shared between devices is routed to the domain once per device,
so vintc_reserve_virq() legitimately fails with -EEXIST for all but the
first of them. map_irq_to_domain() now accepts -EEXIST only for a vIRQ
which is a h/w source already, so that a vIRQ reserved for a virtual
source can't be taken over by a h/w IRQ.

Signed-off-by: Oleksii Kurochko <oleksii.kurochko@xxxxxxxxx>
---
More context on why the two kinds of sources have to be told apart is
given by the patches relying on hw_irqs:
 - "xen/riscv: implement virtual APLIC MMIO emulation", the next one,
   which accesses the registers of a source in the h/w APLIC only for a
   h/w source;
 - "xen/riscv: introduce vaplic_reconfigure_target()", which retargets
   the h/w sources only;
 - the patches introducing virtual sources to vAPLIC:
   "xen/riscv: emulate vAPLIC sourcecfg[], setie[] and domaincfg.IE",
   "xen/riscv: emulate target[] of virtual vAPLIC sources",
   "xen/riscv: emulate the pending bits and MSIs of virtual vAPLIC
   sources" and "xen/riscv: drive the line of a virtual vAPLIC source".
---
Changes in v4:
 - New patch.
---
---
 xen/arch/riscv/device.c           | 17 ++++++++++++-----
 xen/arch/riscv/include/asm/intc.h |  5 ++++-
 xen/arch/riscv/intc.c             | 26 +++++++++++++++++++-------
 xen/arch/riscv/irq.c              |  2 +-
 4 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/xen/arch/riscv/device.c b/xen/arch/riscv/device.c
index fc41c075c772..4c1e18b83c3a 100644
--- a/xen/arch/riscv/device.c
+++ b/xen/arch/riscv/device.c
@@ -25,14 +25,21 @@ int __overlay_init map_irq_to_domain(struct domain *d, 
unsigned int irq,
     {
         /*
          * -EEXIST merely means that the IRQ has already been reserved, which
-         * legitimately happens when the IRQ is shared between devices. Any
-         * other failure has to be fatal: the IRQ would otherwise be routed to
-         * the domain without domain_vintc_deinit() ever releasing it again.
+         * legitimately happens when the IRQ is shared between devices: the
+         * vIRQ is then a h/w source already, route_irq_to_guest() having
+         * connected it to the IRQ for the first of them. A vIRQ reserved for
+         * a virtual source, whose line an emulator in Xen drives, mustn't be
+         * taken over by a h/w IRQ, though. Any other failure has to be fatal
+         * too: the IRQ would otherwise be routed to the domain without
+         * domain_vintc_deinit() ever releasing it again.
          */
         res = vintc_reserve_virq(d, irq);
-        if ( res && (res != -EEXIST) )
+        if ( res == -EEXIST && test_bit(irq, d->arch.vintc->hw_irqs) )
+            res = 0;
+        if ( res )
         {
-            printk(XENLOG_ERR "Unable to reserve vIRQ %u for %pd\n", irq, d);
+            printk(XENLOG_ERR "Unable to reserve vIRQ %u for %pd: %d\n",
+                   irq, d, res);
             return res;
         }
 
diff --git a/xen/arch/riscv/include/asm/intc.h 
b/xen/arch/riscv/include/asm/intc.h
index 909317335769..7a31c6a298a0 100644
--- a/xen/arch/riscv/include/asm/intc.h
+++ b/xen/arch/riscv/include/asm/intc.h
@@ -69,6 +69,8 @@ struct vintc_ops {
 struct vintc {
     unsigned int nr_virqs;
     unsigned long *used_irqs;
+    /* Sources of used_irqs a h/w IRQ is routed to; the rest are virtual. */
+    unsigned long *hw_irqs;
     /* Callbacks invoked during domain construction only. */
     const struct vintc_init_ops *init_ops;
     /* Runtime callbacks used for the lifetime of the guest. */
@@ -82,7 +84,8 @@ void register_intc_ops(const struct intc_hw_init_ops 
*init_ops);
 void intc_init(void);
 
 void intc_route_irq_to_xen(struct irq_desc *desc, unsigned int priority);
-int intc_route_irq_to_guest(struct irq_desc *desc, unsigned int priority);
+int intc_route_irq_to_guest(const struct domain *d, unsigned int virq,
+                            struct irq_desc *desc, unsigned int priority);
 
 void intc_handle_external_irqs(struct cpu_user_regs *regs);
 
diff --git a/xen/arch/riscv/intc.c b/xen/arch/riscv/intc.c
index 810d126e263d..ac9c798ec36b 100644
--- a/xen/arch/riscv/intc.c
+++ b/xen/arch/riscv/intc.c
@@ -82,19 +82,25 @@ void intc_route_irq_to_xen(struct irq_desc *desc, unsigned 
int priority)
     intc_set_irq_priority(desc, priority);
 }
 
-int intc_route_irq_to_guest(struct irq_desc *desc,
-                            unsigned int priority)
+int intc_route_irq_to_guest(const struct domain *d, unsigned int virq,
+                            struct irq_desc *desc, unsigned int priority)
 {
     ASSERT(spin_is_locked(&desc->lock));
 
     ASSERT(intc_hw_ops->guest_irq_type);
 
+    /* The caller has reserved @virq, see vintc_reserve_virq(). */
+    ASSERT(virq < d->arch.vintc->nr_virqs);
+
     desc->handler = intc_hw_ops->guest_irq_type;
     desc->status |= IRQ_GUEST;
 
     intc_set_irq_type(desc, desc->arch.type);
     intc_set_irq_priority(desc, priority);
 
+    /* Connect @virq to the h/w IRQ: it isn't a virtual source. */
+    set_bit(virq, d->arch.vintc->hw_irqs);
+
     return 0;
 }
 
@@ -126,7 +132,10 @@ int domain_vintc_init(struct domain *d)
         d->arch.vintc->used_irqs =
             xvzalloc_array(unsigned long,
                            BITS_TO_LONGS(d->arch.vintc->nr_virqs));
-        if ( !d->arch.vintc->used_irqs )
+        d->arch.vintc->hw_irqs =
+            xvzalloc_array(unsigned long,
+                           BITS_TO_LONGS(d->arch.vintc->nr_virqs));
+        if ( !d->arch.vintc->used_irqs || !d->arch.vintc->hw_irqs )
             ret = -ENOMEM;
     }
 
@@ -140,17 +149,19 @@ void domain_vintc_deinit(struct domain *d)
     if ( !d->arch.vintc )
         return;
 
-    if ( d->arch.vintc->used_irqs )
+    if ( d->arch.vintc->hw_irqs )
     {
         unsigned int virq;
 
+        /* A virtual source has no h/w IRQ routed to the domain to release. */
         for ( virq = 0; virq < d->arch.vintc->nr_virqs; virq++ )
-            if ( test_bit(virq, d->arch.vintc->used_irqs) )
+            if ( test_bit(virq, d->arch.vintc->hw_irqs) )
                 release_guest_irq(d, virq);
-
-        XVFREE(d->arch.vintc->used_irqs);
     }
 
+    XVFREE(d->arch.vintc->used_irqs);
+    XVFREE(d->arch.vintc->hw_irqs);
+
     switch ( variant )
     {
     case INTC_APLIC:
@@ -169,6 +180,7 @@ void domain_vintc_deinit(struct domain *d)
  * Returns 0 on success, -EEXIST if @virq has already been reserved, which
  * legitimately happens when an IRQ is shared between devices, and -ERANGE if
  * @virq is outside the range of the interrupt sources the vINTC provides.
+ * A virtual source can't be shared, so for one -EEXIST is a failure.
  */
 int __overlay_init vintc_reserve_virq(const struct domain *d,
                                       unsigned int virq)
diff --git a/xen/arch/riscv/irq.c b/xen/arch/riscv/irq.c
index 9fc5a1bb703c..1b07b811b1fd 100644
--- a/xen/arch/riscv/irq.c
+++ b/xen/arch/riscv/irq.c
@@ -463,7 +463,7 @@ int route_irq_to_guest(struct domain *d, unsigned int virq,
     if ( retval )
         goto out;
 
-    retval = intc_route_irq_to_guest(desc, IRQ_NO_PRIORITY);
+    retval = intc_route_irq_to_guest(d, virq, desc, IRQ_NO_PRIORITY);
     if ( retval )
     {
         struct irqaction *action = irq_detach_action(desc, info);
-- 
2.56.1