[PATCH v4 07/28] x86: XFD enabling

[Jan Beulich <jbeulich@xxxxxxxx>]

Just like for XCR0 we generally want to run with the guest settings of
the involved MSRs while in Xen. Hence saving/restoring of guest values
needs to only happen during context switch.

Due to the way XFD_ERR is updated by the hardware when #NM is
intercepted, we need to keep XFD_ERR intercepted as well when #NM is.
This way (like for PV) we can properly determine whether a particular
instance of #NM was due to a new bit in the MSR having become set, or
because of lazy-FPU handling.

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
v4: Properly add the two MSRs to all four lists of MSRs to migrate
    (instead of just one of them). Move XFD_ERR handling to
    *_[gs]et_reg(), adjusting it for VMX. Also fiddle with
    do_device_not_available() and vmx_vmexit_handler(). Don't write
    XFD_ERR from paravirt_ctxt_switch_to(). Clear XFD_ERR in
    vmx_save_guest_msrs(). Tie XFD_ERR interception to that of #NM.
    Use wrmsrns(). Re-base.
v3: New.
---
I wasn't sure whether we want to have AMX depend on XFD. Purely from a
spec pov the two are independent features (without XFD one simply
wouldn't have a way to disable it for the purpose of lazy allocation of
state saving space).

TBD: Formally #NM injection in do_device_not_available() may need to use
     struct x86_event's data field, yet it's not quite clear to me whether
     that's really necessary there.

--- a/xen/arch/x86/cpu-policy.c
+++ b/xen/arch/x86/cpu-policy.c
@@ -241,6 +241,9 @@ static void recalculate_xstate(struct cp
         p->xstate.xss_high  = (xstates & XSTATE_XSAVES_ONLY) >> 32;
     }
 
+    if ( p->xstate.xfd )
+        ecx_mask |= XSTATE_XFD;
+
     /* Subleafs 2+ */
     xstates &= ~XSTATE_FP_SSE;
     BUILD_BUG_ON(ARRAY_SIZE(p->xstate.comp) < 63);
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -2036,6 +2036,12 @@ void cf_check paravirt_ctxt_switch_to(st
 
     if ( cpu_has_msr_tsc_aux )
         wrmsr_tsc_aux(v->arch.msrs->tsc_aux);
+
+    if ( v->domain->arch.cpuid->xstate.xfd )
+    {
+        wrmsrns(MSR_XFD, v->arch.msrs->xfd);
+        /* XFD_ERR is kept at zero at all times when Xen is "owning" the MSR. 
*/
+    }
 }
 
 static void _update_runstate_area(struct vcpu *v)
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1153,6 +1153,8 @@ long arch_do_domctl(
         static const uint32_t msrs_to_send[] = {
             MSR_SPEC_CTRL,
             MSR_INTEL_MISC_FEATURES_ENABLES,
+            MSR_XFD,
+            MSR_XFD_ERR,
             MSR_TSC_AUX,
             MSR_AMD64_DR0_ADDRESS_MASK,
             MSR_AMD64_DR1_ADDRESS_MASK,
@@ -1250,6 +1252,8 @@ long arch_do_domctl(
                 {
                 case MSR_SPEC_CTRL:
                 case MSR_INTEL_MISC_FEATURES_ENABLES:
+                case MSR_XFD:
+                case MSR_XFD_ERR:
                 case MSR_TSC_AUX:
                 case MSR_AMD64_DR0_ADDRESS_MASK:
                 case MSR_AMD64_DR1_ADDRESS_MASK ... MSR_AMD64_DR3_ADDRESS_MASK:
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -1396,6 +1396,8 @@ static const uint32_t msrs_to_send[] = {
     MSR_USER_MSR_CTL,
     MSR_SPEC_CTRL,
     MSR_INTEL_MISC_FEATURES_ENABLES,
+    MSR_XFD,
+    MSR_XFD_ERR,
     MSR_PKRS,
     MSR_IA32_BNDCFGS,
     MSR_IA32_XSS,
@@ -1551,6 +1553,8 @@ static int cf_check hvm_load_cpu_msrs(st
         case MSR_USER_MSR_CTL:
         case MSR_SPEC_CTRL:
         case MSR_INTEL_MISC_FEATURES_ENABLES:
+        case MSR_XFD:
+        case MSR_XFD_ERR:
         case MSR_PKRS:
         case MSR_IA32_BNDCFGS:
         case MSR_IA32_XSS:
--- a/xen/arch/x86/hvm/vmx/vmcs.c
+++ b/xen/arch/x86/hvm/vmx/vmcs.c
@@ -1210,6 +1210,12 @@ static int construct_vmcs(struct vcpu *v
         if ( (vmexit_ctl & VM_EXIT_CLEAR_BNDCFGS) &&
              (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS) )
             vmx_clear_msr_intercept(v, MSR_IA32_BNDCFGS, VMX_MSR_RW);
+
+        if ( d->arch.cpuid->xstate.xfd )
+        {
+            vmx_clear_msr_intercept(v, MSR_XFD, VMX_MSR_RW);
+            vmx_clear_msr_intercept(v, MSR_XFD_ERR, VMX_MSR_RW);
+        }
     }
 
     /* I/O access bitmap. */
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -729,6 +729,14 @@ static void vmx_save_guest_msrs(struct v
      */
     v->arch.hvm.vmx.shadow_gs = read_gs_shadow();
 
+    if ( v->domain->arch.cpuid->xstate.xfd )
+    {
+        msrs->xfd = rdmsr(MSR_XFD);
+        msrs->xfd_err = rdmsr(MSR_XFD_ERR);
+        if ( msrs->xfd_err )
+            wrmsrns(MSR_XFD_ERR, 0);
+    }
+
 #ifdef CONFIG_VMTRACE
     if ( v->arch.hvm.vmx.ipt_active )
     {
@@ -754,6 +762,13 @@ static void vmx_restore_guest_msrs(struc
     if ( cpu_has_msr_tsc_aux )
         wrmsr_tsc_aux(msrs->tsc_aux);
 
+    if ( v->domain->arch.cpuid->xstate.xfd )
+    {
+        wrmsrns(MSR_XFD, msrs->xfd);
+        if ( msrs->xfd_err )
+            wrmsrns(MSR_XFD_ERR, msrs->xfd_err);
+    }
+
 #ifdef CONFIG_VMTRACE
     if ( v->arch.hvm.vmx.ipt_active )
     {
@@ -2780,6 +2795,12 @@ static uint64_t cf_check vmx_get_reg(str
     case MSR_PKRS:
         return (v == curr) ? rdpkrs() : msrs->pkrs;
 
+    case MSR_XFD_ERR:
+        if ( v != curr )
+            return msrs->xfd_err;
+        val = rdmsr(MSR_XFD_ERR);
+        return val;
+
     case MSR_SHADOW_GS_BASE:
         if ( v != curr )
             return v->arch.hvm.vmx.shadow_gs;
@@ -2840,6 +2861,12 @@ static void cf_check vmx_set_reg(struct
         if ( v == curr )
             wrpkrs(val);
         return;
+
+    case MSR_XFD_ERR:
+        msrs->xfd_err = val;
+        if ( v == curr )
+            wrmsrns(MSR_XFD_ERR, val);
+        return;
     }
 
     /* Logic which maybe requires remote VMCS acquisition. */
--- a/xen/arch/x86/include/asm/guest-msr.h
+++ b/xen/arch/x86/include/asm/guest-msr.h
@@ -64,6 +64,12 @@ struct vcpu_msrs
         };
     } misc_features_enables;
 
+    /* 0x000001c4 - MSR_XFD */
+    uint64_t xfd;
+
+    /* 0x000001c5 - MSR_XFD_ERR */
+    uint64_t xfd_err;
+
 #ifdef CONFIG_VMTRACE
     /*
      * 0x00000560 ... 57x - MSR_RTIT_*
--- a/xen/arch/x86/include/asm/msr-index.h
+++ b/xen/arch/x86/include/asm/msr-index.h
@@ -124,6 +124,10 @@
 #define MSR_TEMPERATURE_TARGET              0x000001a2
 #define MSR_PACKAGE_THERM_STATUS            0x000001b1
 
+/* Bits within these two MSRs share positions with XCR0. */
+#define MSR_XFD                             0x000001c4
+#define MSR_XFD_ERR                         0x000001c5
+
 #define MSR_FRED_RSP_SL0                    0x000001cc
 #define MSR_FRED_RSP_SL1                    0x000001cd
 #define MSR_FRED_RSP_SL2                    0x000001ce
--- a/xen/arch/x86/include/asm/xstate.h
+++ b/xen/arch/x86/include/asm/xstate.h
@@ -38,9 +38,11 @@ extern uint32_t mxcsr_mask;
 
 #define XSTATE_XSS     (1U << 0)
 #define XSTATE_ALIGN64 (1U << 1)
+#define XSTATE_XFD     (1U << 2)
 
 extern u64 xfeature_mask;
 extern u64 xstate_align;
+extern uint64_t xfd_mask;
 
 #define xstate_offset(n) (raw_cpu_policy.xstate.comp[n].offset)
 #define xstate_size(n)   (raw_cpu_policy.xstate.comp[n].size)
--- a/xen/arch/x86/msr.c
+++ b/xen/arch/x86/msr.c
@@ -220,6 +220,17 @@ int guest_rdmsr(struct vcpu *v, uint32_t
         *val = msrs->misc_features_enables.raw;
         break;
 
+    case MSR_XFD:
+        if ( !cp->xstate.xfd )
+            goto gp_fault;
+        *val = v == curr && is_hvm_domain(d) ? rdmsr(msr) : msrs->xfd;
+        break;
+
+    case MSR_XFD_ERR:
+        if ( !cp->xstate.xfd )
+            goto gp_fault;
+        goto get_reg;
+
     case MSR_P5_MC_ADDR:
     case MSR_P5_MC_TYPE:
     case MSR_IA32_MCG_CAP     ... MSR_IA32_MCG_CTL:      /* 0x179 -> 0x17b */
@@ -549,6 +560,21 @@ int guest_wrmsr(struct vcpu *v, uint32_t
         break;
     }
 
+    case MSR_XFD:
+        if ( !cp->xstate.xfd ||
+             (val & ~(cpu_policy_xstates(cp) & xfd_mask)) )
+            goto gp_fault;
+        if ( v == curr )
+            wrmsrns(msr, val);
+        msrs->xfd = val;
+        break;
+
+    case MSR_XFD_ERR:
+        if ( !cp->xstate.xfd ||
+             (val & ~(cpu_policy_xstates(cp) & xfd_mask)) )
+            goto gp_fault;
+        goto set_reg;
+
     case MSR_IA32_MCG_CAP     ... MSR_IA32_MCG_CTL:      /* 0x179 -> 0x17b */
     case MSR_IA32_MCx_CTL2(0) ... MSR_IA32_MCx_CTL2(31): /* 0x280 -> 0x29f */
     case MSR_IA32_MCx_CTL(0)  ... MSR_IA32_MCx_MISC(31): /* 0x400 -> 0x47f */
--- a/xen/arch/x86/pv/emulate.c
+++ b/xen/arch/x86/pv/emulate.c
@@ -102,6 +102,9 @@ uint64_t pv_get_reg(struct vcpu *v, unsi
     case MSR_SPEC_CTRL:
         return msrs->spec_ctrl.raw;
 
+    case MSR_XFD_ERR:
+        return msrs->xfd_err;
+
     default:
         printk(XENLOG_G_ERR "%s(%pv, 0x%08x) Bad register\n",
                __func__, v, reg);
@@ -123,6 +126,10 @@ void pv_set_reg(struct vcpu *v, unsigned
         msrs->spec_ctrl.raw = val;
         break;
 
+    case MSR_XFD_ERR:
+        msrs->xfd_err = val;
+        break;
+
     default:
         printk(XENLOG_G_ERR "%s(%pv, 0x%08x, 0x%016"PRIx64") Bad register\n",
                __func__, v, reg, val);
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -36,6 +36,7 @@
 #include <asm/flushtlb.h>
 #include <asm/fsgsbase.h>
 #include <asm/gdbsx.h>
+#include <asm/guest-msr.h>
 #include <asm/i387.h>
 #include <asm/io.h>
 #include <asm/irq-vectors.h>
@@ -2019,6 +2020,22 @@ void asmlinkage do_device_not_available(
     }
 
 #ifdef CONFIG_PV
+    if ( curr->arch.msrs->xfd )
+    {
+        uint64_t xfd_err = rdmsr(MSR_XFD_ERR);
+
+        if ( xfd_err )
+        {
+            curr->arch.msrs->xfd_err |= xfd_err;
+
+            /* Bits in XFD_ERR are sticky; clear the MSR right away. */
+            wrmsrns(MSR_XFD_ERR, 0);
+
+            pv_inject_hw_exception(X86_EXC_NM, X86_EVENT_NO_EC);
+            return;
+        }
+    }
+
     if ( !(curr->arch.pv.ctrlreg[0] & X86_CR0_TS) )
     {
         ASSERT_UNREACHABLE();
--- a/xen/arch/x86/xstate.c
+++ b/xen/arch/x86/xstate.c
@@ -30,6 +30,10 @@
 u64 __read_mostly xfeature_mask;
 
 u64 __read_mostly xstate_align;
+
+/* Mask of XSAVE/XRSTOR features supporting XFD. */
+uint64_t __ro_after_init xfd_mask;
+
 static unsigned int __read_mostly xstate_features;
 
 uint32_t __read_mostly mxcsr_mask = 0x0000ffbf;
@@ -114,6 +118,11 @@ static int setup_xstate_features(bool bs
             __set_bit(leaf, &xstate_align);
         else
             BUG_ON(!(ecx & XSTATE_ALIGN64) != !test_bit(leaf, &xstate_align));
+
+        if ( bsp && (ecx & XSTATE_XFD) )
+            __set_bit(leaf, &xfd_mask);
+        else
+            BUG_ON(!(ecx & XSTATE_XFD) != !test_bit(leaf, &xfd_mask));
     }
 
     return 0;
--- a/xen/include/xen/lib/x86/cpu-policy.h
+++ b/xen/include/xen/lib/x86/cpu-policy.h
@@ -265,7 +265,7 @@ struct cpu_policy
         /* Per-component common state.  Valid for i >= 2. */
         struct xstate_component {
             uint32_t size, offset;
-            bool xss:1, align:1;
+            bool xss:1, align:1, xfd:1;
             uint32_t _res_d;
         } comp[CPUID_GUEST_NR_XSTATE];
     } xstate;
--- a/xen/tools/gen-cpuid.py
+++ b/xen/tools/gen-cpuid.py
@@ -259,7 +259,7 @@ def crunch_numbers(state):
         # are instructions built on top of base XSAVE, while others are new
         # instruction groups which are specified to require XSAVE for state
         # management.
-        XSAVE: [XSAVEOPT, XSAVEC, XGETBV1, XSAVES,
+        XSAVE: [XSAVEOPT, XSAVEC, XGETBV1, XSAVES, XFD,
                 AVX, MPX, PKU, AMX_TILE, APX_F, LWP],
 
         # AVX is taken to mean hardware support for 256bit registers (which in