From: Denis Mukhin <dmukhin@xxxxxxxx>
Enabling AVIC implicitly disables the V_IRQ, V_INTR_PRIO, V_IGN_TPR,
and V_INTR_VECTOR fields in the VMCB Control Word.
Introduce new interrupt injection code via AVIC backing page.
Essentially, that is handling of 'IsRunning' bit in physical APIC ID
table entry: it needs to be in sync with the scheduled vCPU. Shadow
the state to work around hardware not perceiving 'IsRunning' bit.
Add new doorbell MSR to accelerate IPI emulation. The MSR can be used
for posting interrupts from hypervisor to the guest (e.g. event
channels).
Signed-off-by: Denis Mukhin <dmukhin@xxxxxxxx>
---
xen/arch/x86/hvm/svm/avic.c | 156 +++++++++++++++++++++++
xen/arch/x86/hvm/svm/avic.h | 17 +++
xen/arch/x86/hvm/svm/intr.c | 9 ++
xen/arch/x86/hvm/svm/svm.c | 32 +++++
xen/arch/x86/include/asm/hvm/svm-types.h | 2 +
xen/arch/x86/include/asm/msr-index.h | 1 +
6 files changed, 217 insertions(+)
diff --git a/xen/arch/x86/hvm/svm/avic.c b/xen/arch/x86/hvm/svm/avic.c
index 18a814871133..0bcab0465227 100644
--- a/xen/arch/x86/hvm/svm/avic.c
+++ b/xen/arch/x86/hvm/svm/avic.c
@@ -33,6 +33,8 @@
#define AVIC_MAX_PHYSICAL_ID (0xFEU)
bool __read_mostly svm_avic_enable;
+unsigned int __read_mostly svm_avic_features = AVIC_HW_ISRUNNING |
+ AVIC_DOORBELL_MSR;
static const char __section(".bss.page_aligned.const") __aligned(PAGE_SIZE)
avic_apic_access_page[PAGE_SIZE];
@@ -50,6 +52,11 @@ static int cf_check avic_parse_cmdline(const char *s)
{
svm_avic_enable = true;
}
+ else if ( !strcmp(s, "nodoorbell") )
+ {
+ svm_avic_enable = true;
+ svm_avic_features &= ~AVIC_DOORBELL_MSR;
+ }
else
{
return -EINVAL;
@@ -228,6 +235,7 @@ int svm_avic_vmcb_create(struct vcpu *v)
entry |= AVIC_PHYSICAL_ID_ENTRY_VALID_MASK;
ACCESS_ONCE(*sv->avic_physical_id) = entry;
+ ACCESS_ONCE(sv->avic_shadow_id) = entry;
avic_vcpu_enable(v, true);
@@ -248,6 +256,102 @@ void svm_avic_vmcb_destroy(struct vcpu *v)
vmcb->avic_vapic_bar = 0;
}
+static void avic_vcpu_set_cpu(struct vcpu *v)
+{
+ struct svm_vcpu *sv = &v->arch.hvm.svm;
+ unsigned apic_id = cpu_data[v->processor].apicid;
+ uint64_t entry;
+
+ BUG_ON(sv->avic_physical_id == NULL);
+ ASSERT(!(apic_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK));
+
+ entry = ACCESS_ONCE(sv->avic_shadow_id);
+ entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
+ entry |= apic_id;
+
+ ACCESS_ONCE(*sv->avic_physical_id) = entry;
+
+ ACCESS_ONCE(sv->avic_shadow_id) = entry;
+}
+
+static void avic_vcpu_set_running(struct vcpu *v)
+{
+ struct svm_vcpu *sv = &v->arch.hvm.svm;
+ uint64_t entry;
+
+ BUG_ON(sv->avic_physical_id == NULL);
+
+ entry = ACCESS_ONCE(sv->avic_shadow_id);
+ entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
+
+ /* Shadow IsRunning state update. */
+ ACCESS_ONCE(sv->avic_shadow_id) = entry;
+
+ /*
+ * If hardware has problems detecting IsRunning (erratum 1235)
+ * keep IsRunning in the shadow entry only.
+ */
+ if ( svm_avic_features & AVIC_HW_ISRUNNING )
+ ACCESS_ONCE(*sv->avic_physical_id) = entry;
+}
+
+static void avic_vcpu_clear_running(struct vcpu *v)
+{
+ struct svm_vcpu *sv = &v->arch.hvm.svm;
+ unsigned apic_id = cpu_data[v->processor].apicid;
+ uint64_t entry;
+
+ BUG_ON(sv->avic_physical_id == NULL);
+ ASSERT(!(apic_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK));
+
+ entry = ACCESS_ONCE(sv->avic_shadow_id);
+ entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
+
+ /*
+ * If hardware has problems detecting IsRunning (erratum 1235)
+ * keep IsRunning in the shadow entry only.
+ */
+ if ( svm_avic_features & AVIC_HW_ISRUNNING )
+ ACCESS_ONCE(*sv->avic_physical_id) = entry;
+
+ /* Shadow IsRunning state update. */
+ ACCESS_ONCE(sv->avic_shadow_id) = entry;
+}
+
+/* vCPU enters blocking state. */
+void svm_avic_vcpu_block(struct vcpu *v)
+{
+ avic_vcpu_clear_running(v);
+}
+
+/* vCPU is preparing to resume guest execution. */
+void svm_avic_vcpu_resume(struct vcpu *v)
+{
+ avic_vcpu_set_running(v);
+}
+
+/* vCPU is being assigned a new pCPU. */
+void svm_avic_vcpu_switch_to(struct vcpu *v)
+{
+ avic_vcpu_set_cpu(v);
+}
+
+/* vCPU is being removed from a pCPU. */
+void svm_avic_vcpu_switch_from(struct vcpu *v)
+{
+ avic_vcpu_clear_running(v);
+}
+
+bool svm_avic_vcpu_enabled(const struct vcpu *v)
+{
+ const struct svm_vcpu *sv = &v->arch.hvm.svm;
+ const struct vmcb_struct *vmcb = sv->vmcb;
+
+ ASSERT(!(vmcb->_vintr.fields.x2avic_enable));
+
+ return vmcb->_vintr.fields.avic_enable;
+}
+
/*
* Handle the AVIC_INCOMP_IPI #VMEXIT when AVIC is enabled.
*
@@ -546,6 +650,58 @@ void svm_avic_vmexit_noaccel(struct cpu_user_regs *regs)
hvm_emulate_one_vm_event(EMUL_KIND_NORMAL, X86_EXC_UD,
X86_EVENT_NO_EC);
}
+static void avic_deliver_posted_intr(struct vcpu *v)
+{
+ struct svm_vcpu *sv = &v->arch.hvm.svm;
+ uint64_t entry;
+ unsigned int apicid;
+
+ smp_mb();
+
+ entry = ACCESS_ONCE(sv->avic_shadow_id);
+
+ if ( !(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK) )
+ return;
+
+ /*
+ * Some AMD systems (e.g. Zen5 Family 26 (0x1a), Model 36 (0x24)) have
+ * broken remote notification via doorbell MSR.
+ * Fallback to brute force, if requested.
+ */
+ if ( svm_avic_features & AVIC_DOORBELL_MSR )
+ {
+ apicid = entry & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
+
+ /* Ensure the pending interrupt wakes vCPU. */
+ vcpu_unblock(v);
+
+ /*
+ * AMD APM, 5.29.8.2 Doorbell Mechanism
+ * The serializing semantics of WRMSR are relaxed when writing to
+ * the Doorbell Register.
+ */
+ wrmsrl(MSR_AMD_AVIC_DOORBELL, apicid);
+ }
+ else
+ {
+ vcpu_kick(v);
+ }
+}
+
+void cf_check svm_avic_hvm_deliver_posted_intr(struct vcpu *v, uint8_t vector)
+{
+ ASSERT(v != current);
+
+ if ( svm_avic_vcpu_enabled(v) )
+ {
+ avic_deliver_posted_intr(v);
+ }
+ else
+ {
+ vcpu_kick(v);
+ }
+}
+
/*
* Local variables:
* mode: C
diff --git a/xen/arch/x86/hvm/svm/avic.h b/xen/arch/x86/hvm/svm/avic.h
index f9ff3ffc9346..098924925e89 100644
--- a/xen/arch/x86/hvm/svm/avic.h
+++ b/xen/arch/x86/hvm/svm/avic.h
@@ -6,6 +6,15 @@
#ifndef __SVM_AVIC_H__
#define __SVM_AVIC_H__
+#include <xen/const.h>
+
+/* Known AVIC hardware quirks. */
+enum {
+ AVIC_HW_ISRUNNING = BIT(0, U), /* See erratum 1235 */
+ AVIC_DOORBELL_MSR = BIT(1, U), /* Remote vCPU notification OK */
+};
+
+extern unsigned int svm_avic_features;
extern bool svm_avic_enable;
int svm_avic_domain_initialise(struct domain *d);
@@ -17,4 +26,12 @@ void svm_avic_vmcb_destroy(struct vcpu *v);
void svm_avic_vmexit_incomp_ipi(struct cpu_user_regs *regs);
void svm_avic_vmexit_noaccel(struct cpu_user_regs *regs);
+bool svm_avic_vcpu_enabled(const struct vcpu *v);
+void svm_avic_vcpu_block(struct vcpu *v);
+void svm_avic_vcpu_resume(struct vcpu *v);
+void svm_avic_vcpu_switch_to(struct vcpu *v);
+void svm_avic_vcpu_switch_from(struct vcpu *v);
+
+void cf_check svm_avic_hvm_deliver_posted_intr(struct vcpu *v, uint8_t vector);
+
#endif /* __SVM_AVIC_H__ */
diff --git a/xen/arch/x86/hvm/svm/intr.c b/xen/arch/x86/hvm/svm/intr.c
index 1d39f25e5209..9cceca704a91 100644
--- a/xen/arch/x86/hvm/svm/intr.c
+++ b/xen/arch/x86/hvm/svm/intr.c
@@ -24,6 +24,7 @@
#include <public/hvm/ioreq.h>
#include <xen/domain_page.h>
+#include "avic.h"
#include "nestedhvm.h"
#include "vmcb.h"
@@ -80,6 +81,14 @@ static void svm_enable_intr_window(struct vcpu *v, struct hvm_intack intack)
ASSERT(intack.source != hvm_intsrc_none);
+ if ( svm_avic_vcpu_enabled(v) )
+ /*
+ * AVIC delivers pending LAPIC interrupts from the backing page
+ * when the guest becomes interruptible, no need to arm the legacy
+ * V_IRQ/VINTR interrupt-window mechanism.
+ */
+ return;
+
