[PATCH v4 13/28] x86emul: support TILELOADD{,T1} and TILESTORED

[Jan Beulich <jbeulich@xxxxxxxx>]

In order to be flexible about future growth of tile dimensions, use a
separately allocated scratch buffer to read/write individual rows of a
tile.

Note that the separate write_tilecfg() function is needed because
LDTILECFG wouldn't serve the purpose: It clears various state, unlike
XRSTOR / XRSTORC. To keep things simple, the test harness variant of it
leverages the state save/restore around library calls. To be sure the
actual data gets restored (and not init state put in place), an extra
override for the XSTATE_BV field is introduced.

Signed-off-by: Jan Beulich <jbeulich@xxxxxxxx>
---
v4: Correct store de-duplication faulting behavior. Reject non-memory
    forms of the encodings. Correct invoke_stub() constraint. Use
    X86_EXC_*. Re-base, in particular also over the earlier CPUID
    adjustment.
v3: New.
---
TBD: ISA extensions document 043 doesn't clarify either way whether
     duplicate stores to the same (linear) memory ranges may get
     squashed by TILESTORED just like is documented for AVX-512
     scatters. Similar behavior was informally confirmed.

--- a/tools/include/xen-tools/common-macros.h
+++ b/tools/include/xen-tools/common-macros.h
@@ -76,6 +76,13 @@
 #define MASK_EXTR(v, m) (((v) & (m)) / ((m) & -(m)))
 #define MASK_INSR(v, m) (((v) * ((m) & -(m))) & (m))
 
+/* Base-2 logarithm */
+#define __L2(x)  (((x) & 0x00000002U) ?   1                     : 0)
+#define __L4(x)  (((x) & 0x0000000cU) ? ( 2 + __L2( (x) >> 2))  : __L2( x))
+#define __L8(x)  (((x) & 0x000000f0U) ? ( 4 + __L4( (x) >> 4))  : __L4( x))
+#define __L16(x) (((x) & 0x0000ff00U) ? ( 8 + __L8( (x) >> 8))  : __L8( x))
+#define ilog2(x) (((x) & 0xffff0000U) ? (16 + __L16((x) >> 16)) : __L16(x))
+
 #ifndef __must_check
 #define __must_check __attribute__((__warn_unused_result__))
 #endif
--- a/tools/tests/x86_emulator/predicates.c
+++ b/tools/tests/x86_emulator/predicates.c
@@ -1347,6 +1347,9 @@ static const struct vex {
     { { 0x49, 0x00 }, 2, F, W, pfx_66, W0, L0 }, /* sttilecfg */
     { { 0x49, 0xc0 }, 2, F, N, pfx_no, W0, L0 }, /* tilerelease */
     { { 0x49, 0xc0 }, 2, F, N, pfx_f2, W0, L0 }, /* tilezero */
+    { { 0x4b, VSIB(0) }, 3, F, R, pfx_66, W0, L0 }, /* tileloaddt1 */
+    { { 0x4b, VSIB(0) }, 3, F, W, pfx_f3, W0, L0 }, /* tilestored */
+    { { 0x4b, VSIB(0) }, 3, F, R, pfx_f2, W0, L0 }, /* tileloadd */
     { { 0x50 }, 2, T, R, pfx_no, W0, Ln }, /* vpdpbuud */
     { { 0x50 }, 2, T, R, pfx_66, W0, Ln }, /* vpdpbusd */
     { { 0x50 }, 2, T, R, pfx_f3, W0, Ln }, /* vpdpbsud */
--- a/tools/tests/x86_emulator/x86-emulate.c
+++ b/tools/tests/x86_emulator/x86-emulate.c
@@ -8,6 +8,8 @@ struct cpu_policy cpu_policy;
 
 static char fpu_save_area[0x4000] __attribute__((__aligned__((64))));
 static bool use_xsave;
+/* This will be OR-ed into xstate_bv prior to restoring state. */
+static uint64_t xstate_bv_or_mask;
 
 /*
  * Re-use the area above also as scratch space for the emulator itself.
@@ -33,13 +35,19 @@ void emul_restore_fpu_state(void)
 {
     /* Older gcc can't deal with "m" array inputs; make them outputs instead. 
*/
     if ( use_xsave )
+    {
+        *(uint64_t *)&fpu_save_area[0x200] |= xstate_bv_or_mask;
+        xstate_bv_or_mask = 0;
         asm volatile ( "xrstor %[ptr]"
                        : [ptr] "+m" (fpu_save_area)
                        : "a" (~0ul), "d" (~0ul) );
+    }
     else
         asm volatile ( "fxrstor %0" : "+m" (fpu_save_area) );
 }
 
+static void *tile_row;
+
 bool emul_test_init(void)
 {
     union {
@@ -101,6 +109,17 @@ bool emul_test_init(void)
     if ( fxs->mxcsr_mask )
         mxcsr_mask = fxs->mxcsr_mask;
 
+    if ( cpu_policy.tile.max_palette >= 1 )
+    {
+        unsigned int max_bytes = cpu_policy.tile.palette1.bytes_per_row;
+
+        if ( !cpu_policy.xstate.comp[ilog2(X86_XCR0_TILE_CFG)].offset )
+            max_bytes = 0;
+
+        if ( !max_bytes || !(tile_row = malloc(max_bytes)) )
+            cpu_policy.feat.amx_tile = false;
+    }
+
     /*
      * Mark the entire stack executable so that the stub executions
      * don't fault
@@ -255,4 +274,22 @@ void wrpkru(unsigned int val)
     pkru = val;
 }
 
+static void *get_tile_row_buf(void)
+{
+    return tile_row;
+}
+
+static void write_tilecfg(const struct x86_tilecfg *tilecfg)
+{
+    /*
+     * Leverage the fact that the wrapper (saves and) restores all extended
+     * state around the actual library call. Invoke the wrapper explicitly to
+     * make sure the compiler won't inline the call.
+     */
+    xstate_bv_or_mask = X86_XCR0_TILE_CFG;
+    __wrap_memcpy((fpu_save_area +
+                   cpu_policy.xstate.comp[ilog2(X86_XCR0_TILE_CFG)].offset),
+                  tilecfg, sizeof(*tilecfg));
+}
+
 #include "x86_emulate/x86_emulate.c"
--- a/xen/arch/x86/include/asm/cpufeature.h
+++ b/xen/arch/x86/include/asm/cpufeature.h
@@ -208,6 +208,7 @@ static inline bool boot_cpu_has(unsigned
 #define cpu_has_rtm_always_abort boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)
 #define cpu_has_tsx_force_abort boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)
 #define cpu_has_hybrid          boot_cpu_has(X86_FEATURE_HYBRID)
+#define cpu_has_amx_tile        boot_cpu_has(X86_FEATURE_AMX_TILE)
 #define cpu_has_arch_caps       boot_cpu_has(X86_FEATURE_ARCH_CAPS)
 
 /* CPUID level 0x00000007:1.eax */
--- a/xen/arch/x86/include/asm/xstate.h
+++ b/xen/arch/x86/include/asm/xstate.h
@@ -94,6 +94,8 @@ uint64_t get_xcr0(void);
 void set_msr_xss(u64 xss);
 uint64_t get_msr_xss(void);
 uint64_t read_bndcfgu(void);
+struct x86_tilecfg;
+void write_tilecfg(const struct x86_tilecfg *tilecfg);
 void xsave(struct vcpu *v, uint64_t mask);
 void xrstor(struct vcpu *v, uint64_t mask);
 void xstate_set_init(uint64_t mask);
--- a/xen/arch/x86/x86_emulate/decode.c
+++ b/xen/arch/x86/x86_emulate/decode.c
@@ -364,6 +364,7 @@ static const struct ext0f38_table {
     [0x44] = { .simd_size = simd_packed_int, .two_op = 1, .d8s = d8s_vl },
     [0x45 ... 0x47] = { .simd_size = simd_packed_int, .d8s = d8s_vl },
     [0x49] = { .simd_size = simd_other, .two_op = 1 },
+    [0x4b] = { .simd_size = simd_other, .two_op = 1, .vsib = 1 },
     [0x4c] = { .simd_size = simd_packed_fp, .two_op = 1, .d8s = d8s_vl },
     [0x4d] = { .simd_size = simd_scalar_vexw, .d8s = d8s_dq },
     [0x4e] = { .simd_size = simd_packed_fp, .two_op = 1, .d8s = d8s_vl },
@@ -1268,6 +1269,7 @@ decode_0f38(struct x86_emulate_state *s,
         break;
 
     case X86EMUL_OPC_VEX_66(0, 0x49): /* sttilecfg */
+    case X86EMUL_OPC_VEX_F3(0, 0x4b): /* tilestored */
         s->desc = DstMem | SrcImplicit | Mov;
         break;
 
--- a/xen/arch/x86/x86_emulate/x86_emulate.c
+++ b/xen/arch/x86/x86_emulate/x86_emulate.c
@@ -97,6 +97,77 @@ static const uint16_t _3dnow_ext_table[1
 /* Shift values between src and dst sizes of pmov{s,z}x{b,w,d}{w,d,q}. */
 static const uint8_t pmov_convert_delta[] = { 1, 2, 3, 1, 2, 1 };
 
+# ifdef __XEN__
+
+#  include <xen/cpu.h>
+#  include <xen/xvmalloc.h>
+
+static DEFINE_PER_CPU_READ_MOSTLY(void *, tile_row);
+static unsigned int __read_mostly tile_row_max_bytes;
+
+static int cf_check amx_cpu_init(struct notifier_block *nfb,
+                                 unsigned long action, void *hcpu)
+{
+    int rc = 0;
+    void **prow = &per_cpu(tile_row, (unsigned long)hcpu);
+
+    switch ( action )
+    {
+    case CPU_UP_PREPARE:
+        *prow = xvmalloc_array(uint8_t, tile_row_max_bytes);
+        if ( !*prow )
+            rc = -ENOMEM;
+        break;
+
+    case CPU_UP_CANCELED:
+    case CPU_DEAD:
+        XVFREE(*prow);
+        break;
+
+    default:
+        /* Notifier pattern */
+        break;
+    }
+
+    return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
+}
+
+static struct notifier_block amx_cpu_nfb = {
+    .notifier_call = amx_cpu_init,
+};
+
+static int __init cf_check amx_init(void)
+{
+    const struct cpu_policy *p = &host_cpu_policy;
+
+    if ( !cpu_has_amx_tile )
+        return 0;
+
+    if ( p->tile.max_palette >= 1 &&
+         p->tile.palette1.bytes_per_row > tile_row_max_bytes )
+        tile_row_max_bytes = p->tile.palette1.bytes_per_row;
+
+    if ( !tile_row_max_bytes )
+    {
+        setup_clear_cpu_cap(X86_FEATURE_AMX_TILE);
+        return 0;
+    }
+
+    amx_cpu_init(&amx_cpu_nfb, CPU_UP_PREPARE,
+                 (void *)(unsigned long)smp_processor_id());
+    register_cpu_notifier(&amx_cpu_nfb);
+
+    return 0;
+}
+presmp_initcall(amx_init);
+
+static void *get_tile_row_buf(void)
+{
+    return this_cpu(tile_row);
+}
+
+# endif /* __XEN__ */
+
 static void sttilecfg(struct x86_tilecfg *tilecfg)
 {
     /* sttilecfg (%rdi) */
@@ -6691,6 +6762,119 @@ x86_emulate(
         }
         goto unrecognized_insn;
 
+    case X86EMUL_OPC_VEX_66(0x0f38, 0x4b): /* tileloaddt1 mem,tmm */
+    case X86EMUL_OPC_VEX_F3(0x0f38, 0x4b): /* tilestored tmm,mem */
+    case X86EMUL_OPC_VEX_F2(0x0f38, 0x4b): /* tileloadd mem,tmm */
+    {
+        struct x86_tilecfg *cfg = &mmvalp->tilecfg;
+        unsigned int row, iter;
+        void *data;
+
+        if ( ea.type != OP_MEM )
+            goto unrecognized_insn;
+        generate_exception_if(!mode_64bit() || vex.l || vex.w || vex.reg != 
0xf,
+                              X86_EXC_UD);
+        vcpu_must_have(amx_tile);
+        get_fpu(X86EMUL_FPU_tile);
+        sttilecfg(cfg);
+        generate_exception_if(!tile_valid(modrm_reg, cfg, PALETTE(AMX)),
+                              X86_EXC_UD);
+        generate_exception_if(cfg->colsb[modrm_reg] & 3, X86_EXC_UD);
+        row = i = cfg->start_row;
+        iter = n = cfg->rows[modrm_reg];
+        generate_exception_if(i >= n, X86_EXC_UD);
+
+        /* Calculate stride. */
+        if ( state->sib_index != 4 )
+            ea.val = truncate_ea(*decode_gpr(ctxt->regs, NULL,
+                                             state->sib_index) <<
+                                 state->sib_scale);
+
+        if ( vex.pfx == vex_f3 && n > 1 )
+        {
+            /*
+             * hvmemul_linear_mmio_access() will find a cache slot based on
+             * linear address.  hvmemul_phys_mmio_access() will crash the
+             * domain if observing varying data getting written to the same
+             * cache slot.  Assume that squashing earlier writes to fully
+             * overlapping addresses is permitted and that later writes may
+             * complete before earlier faults (like for AVX-512 scatters).
+             * To maintain correct faulting behavior we need to report a low
+             * enough start_row in the event of an exception.
+             */
+            if ( !ea.val )
+                row = n - (iter = 1);
+            else if ( !(ea.val &
+                        ((1ul << ((ad_bytes - sizeof(*cfg->rows)) * 8)) - 1)) )
+            {
+                unsigned int clr = __builtin_ffsl(ea.val) - 1;
+
+                iter = 1u << (ad_bytes * 8 - clr);
+                if ( iter < n - i )
+                    row = n - (n % iter ?: iter);
+                else
+                    iter = n;
+            }
+        }
+
+        data = get_tile_row_buf();
+        switch ( cfg->palette )
+        {
+        case 1: /* AMX */
+            memset(data, -1, cp->tile.palette1.bytes_per_row);
+            break;
+
+        default:
+            ASSERT_UNREACHABLE();
+            generate_exception(X86_EXC_GP, 0);
+        }
+
+        /* Set up stub. */
+        opc = init_prefixes(stub);
+        opc[0] = b;
+        /* Convert memory operand to (%rax,%riz,1) */
+        vex.b = 1;
+        vex.x = 1;
+        opc[1] = modrm & 0x3f;
+        opc[2] = 0x20;
+        place_ret(&opc[3]);
+        copy_VEX(opc, vex, false);
+
+        do {
+            if ( row >= n )
+                row -= iter;
+            /* Limit rows to just as many to cover the next one to access. */
+            cfg->start_row = row;
+            cfg->rows[modrm_reg] = row + 1;
+            write_tilecfg(cfg);
+
+            if ( vex.pfx != vex_f3 )
+                rc = ops->read(ea.mem.seg,
+                               truncate_ea(ea.mem.off + i * ea.val),
+                               data, cfg->colsb[modrm_reg], ctxt);
+
+            invoke_stub("", "", "+m" (*(uint8_t(*)[64])data) : "a" (data));
+
+            if ( vex.pfx == vex_f3 )
+                rc = ops->write(ea.mem.seg,
+                                truncate_ea(ea.mem.off + i * ea.val),
+                                data, cfg->colsb[modrm_reg], ctxt);
+        } while ( rc == X86EMUL_OKAY && (++row, ++i < iter) );
+
+        put_stub(stub);
+
+        if ( rc == X86EMUL_OKAY )
+            cfg->start_row = 0;
+        else if ( vex.pfx == vex_f3 )
+            cfg->start_row = i;
+        cfg->rows[modrm_reg] = n;
+        write_tilecfg(cfg);
+
+        state->simd_size = simd_none;
+        dst.type = OP_NONE;
+        break;
+    }
+
     case X86EMUL_OPC_VEX   (0x0f38, 0x50): /* vpdpbuud 
[xy]mm/mem,[xy]mm,[xy]mm */
     case X86EMUL_OPC_VEX_F3(0x0f38, 0x50): /* vpdpbsud 
[xy]mm/mem,[xy]mm,[xy]mm */
     case X86EMUL_OPC_VEX_F2(0x0f38, 0x50): /* vpdpbssd 
[xy]mm/mem,[xy]mm,[xy]mm */
--- a/xen/arch/x86/xstate.c
+++ b/xen/arch/x86/xstate.c
@@ -16,6 +16,7 @@
 #include <asm/i387.h>
 #include <asm/msr.h>
 #include <asm/processor.h>
+#include <asm/x86-types.h>
 #include <asm/xstate.h>
 
 /*
@@ -1082,6 +1083,37 @@ uint64_t read_bndcfgu(void)
     return xstate->xsave_hdr.xstate_bv & X86_XCR0_BNDCSR ? bndcsr->bndcfgu : 0;
 }
 
+void write_tilecfg(const struct x86_tilecfg *tilecfg)
+{
+    unsigned long cr0 = read_cr0();
+    struct xsave_struct *xstate
+        = idle_vcpu[smp_processor_id()]->arch.xsave_area;
+
+    ASSERT(cpu_has_amx_tile);
+    clts();
+
+    memset(&xstate->xsave_hdr, 0, sizeof(xstate->xsave_hdr));
+    xstate->xsave_hdr.xstate_bv = X86_XCR0_TILE_CFG;
+
+    if ( cpu_has_xsavec )
+    {
+        xstate->xsave_hdr.xcomp_bv = XSTATE_COMPACTION_ENABLED |
+                                     X86_XCR0_TILE_CFG;
+
+        memcpy(xstate + 1, tilecfg, sizeof(*tilecfg));
+    }
+    else
+        memcpy((void *)xstate + xstate_offset(ilog2(X86_XCR0_TILE_CFG)),
+               tilecfg, sizeof(*tilecfg));
+
+    asm volatile ( ".byte 0x0f,0xae,0x2f\n" /* xrstor */
+                   :: "m" (*xstate), "D" (xstate),
+                      "a" (X86_XCR0_TILE_CFG), "d" (0) );
+
+    if ( cr0 & X86_CR0_TS )
+        write_cr0(cr0);
+}
+
 void xstate_set_init(uint64_t mask)
 {
     unsigned long cr0 = read_cr0();