# HG changeset patch
# User Tim Deegan <Tim.Deegan@xxxxxxxxxxxxx>
# Date 1180705852 -3600
# Node ID bd3d6b4c52ec809f080c89c4ffcf61dc6e445978
# Parent 13eca4bf2c69aff4c9b689d0dff45929e44e4edb
[XEN] Shadow: cache gva->gfn+rights translations between guest TLB flushes
to make lookups faster for emulation and hvm_copy.
Signed-off-by: Tim Deegan <Tim.Deegan@xxxxxxxxxxxxx>
---
xen/arch/x86/mm/shadow/common.c | 30 ++++++++++++++++
xen/arch/x86/mm/shadow/multi.c | 66 ++++++++++++++++++++++++++++++++----
xen/arch/x86/mm/shadow/private.h | 71 +++++++++++++++++++++++++++++++++++++--
xen/arch/x86/mm/shadow/types.h | 3 +
xen/include/asm-x86/domain.h | 3 +
5 files changed, 164 insertions(+), 9 deletions(-)
diff -r 13eca4bf2c69 -r bd3d6b4c52ec xen/arch/x86/mm/shadow/common.c
--- a/xen/arch/x86/mm/shadow/common.c Fri Jun 01 14:32:11 2007 +0100
+++ b/xen/arch/x86/mm/shadow/common.c Fri Jun 01 14:50:52 2007 +0100
@@ -2206,6 +2206,24 @@ static void sh_update_paging_modes(struc
ASSERT(shadow_locked_by_me(d));
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ /* Make sure this vcpu has a virtual TLB array allocated */
+ if ( unlikely(!v->arch.paging.vtlb) )
+ {
+ v->arch.paging.vtlb = xmalloc_array(struct shadow_vtlb, VTLB_ENTRIES);
+ if ( unlikely(!v->arch.paging.vtlb) )
+ {
+ SHADOW_ERROR("Could not allocate vTLB space for dom %u vcpu %u\n",
+ d->domain_id, v->vcpu_id);
+ domain_crash(v->domain);
+ return;
+ }
+ memset(v->arch.paging.vtlb, 0,
+ VTLB_ENTRIES * sizeof (struct shadow_vtlb));
+ spin_lock_init(&v->arch.paging.vtlb_lock);
+ }
+#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
+
// Valid transitions handled by this function:
// - For PV guests:
// - after a shadow mode has been changed
@@ -2513,6 +2531,18 @@ void shadow_teardown(struct domain *d)
}
}
}
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ /* Free the virtual-TLB array attached to each vcpu */
+ for_each_vcpu(d, v)
+ {
+ if ( v->arch.paging.vtlb )
+ {
+ xfree(v->arch.paging.vtlb);
+ v->arch.paging.vtlb = NULL;
+ }
+ }
+#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
list_for_each_safe(entry, n, &d->arch.paging.shadow.p2m_freelist)
{
diff -r 13eca4bf2c69 -r bd3d6b4c52ec xen/arch/x86/mm/shadow/multi.c
--- a/xen/arch/x86/mm/shadow/multi.c Fri Jun 01 14:32:11 2007 +0100
+++ b/xen/arch/x86/mm/shadow/multi.c Fri Jun 01 14:50:52 2007 +0100
@@ -2997,6 +2997,11 @@ sh_invlpg(struct vcpu *v, unsigned long
perfc_incr(shadow_invlpg);
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ /* No longer safe to use cached gva->gfn translations */
+ vtlb_flush(v);
+#endif
+
/* First check that we can safely read the shadow l2e. SMP/PAE linux can
* run as high as 6% of invlpg calls where we haven't shadowed the l2
* yet. */
@@ -3057,6 +3062,7 @@ sh_invlpg(struct vcpu *v, unsigned long
return 1;
}
+
static unsigned long
sh_gva_to_gfn(struct vcpu *v, unsigned long va)
/* Called to translate a guest virtual address to what the *guest*
@@ -3064,11 +3070,24 @@ sh_gva_to_gfn(struct vcpu *v, unsigned l
{
walk_t gw;
gfn_t gfn;
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ struct shadow_vtlb t = {0};
+ if ( vtlb_lookup(v, va, &t) )
+ return t.frame_number;
+#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
guest_walk_tables(v, va, &gw, 0);
gfn = guest_walk_to_gfn(&gw);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ t.page_number = va >> PAGE_SHIFT;
+ t.frame_number = gfn_x(gfn);
+ t.flags = accumulate_guest_flags(v, &gw);
+ vtlb_insert(v, t);
+#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
+
unmap_walk(v, &gw);
-
return gfn_x(gfn);
}
@@ -3694,6 +3713,11 @@ sh_update_cr3(struct vcpu *v, int do_loc
/* Fix up the linear pagetable mappings */
sh_update_linear_entries(v);
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ /* No longer safe to use cached gva->gfn translations */
+ vtlb_flush(v);
+#endif
+
/* Release the lock, if we took it (otherwise it's the caller's problem) */
if ( do_locking ) shadow_unlock(v->domain);
}
@@ -3918,13 +3942,41 @@ static inline void * emulate_map_dest(st
if ( ring_3(sh_ctxt->ctxt.regs) )
return NULL;
- /* Walk the guest pagetables */
- guest_walk_tables(v, vaddr, &gw, 1);
- flags = accumulate_guest_flags(v, &gw);
- gfn = guest_l1e_get_gfn(gw.eff_l1e);
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ /* Try the virtual TLB first */
+ {
+ struct shadow_vtlb t = {0};
+ if ( vtlb_lookup(v, vaddr, &t)
+ && ((t.flags & (_PAGE_PRESENT|_PAGE_RW))
+ == (_PAGE_PRESENT|_PAGE_RW)) )
+ {
+ flags = t.flags;
+ gfn = _gfn(t.frame_number);
+ }
+ else
+ {
+ /* Need to do the full lookup, just in case permissions
+ * have increased since we cached this entry */
+
+#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
+
+ /* Walk the guest pagetables */
+ guest_walk_tables(v, vaddr, &gw, 1);
+ flags = accumulate_guest_flags(v, &gw);
+ gfn = guest_l1e_get_gfn(gw.eff_l1e);
+ sh_audit_gw(v, &gw);
+ unmap_walk(v, &gw);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+ /* Remember this translation for next time */
+ t.page_number = vaddr >> PAGE_SHIFT;
+ t.frame_number = gfn_x(gfn);
+ t.flags = flags;
+ vtlb_insert(v, t);
+ }
+ }
+#endif
mfn = vcpu_gfn_to_mfn(v, gfn);
- sh_audit_gw(v, &gw);
- unmap_walk(v, &gw);
errcode = PFEC_write_access;
if ( !(flags & _PAGE_PRESENT) )
diff -r 13eca4bf2c69 -r bd3d6b4c52ec xen/arch/x86/mm/shadow/private.h
--- a/xen/arch/x86/mm/shadow/private.h Fri Jun 01 14:32:11 2007 +0100
+++ b/xen/arch/x86/mm/shadow/private.h Fri Jun 01 14:50:52 2007 +0100
@@ -61,8 +61,9 @@ extern int shadow_audit_enable;
#define SHOPT_PREFETCH 0x08 /* Shadow multiple entries per fault */
#define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */
#define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */
-
-#define SHADOW_OPTIMIZATIONS 0x3f
+#define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */
+
+#define SHADOW_OPTIMIZATIONS 0x7f
/******************************************************************************
@@ -649,6 +650,72 @@ void shadow_continue_emulation(
void shadow_continue_emulation(
struct sh_emulate_ctxt *sh_ctxt, struct cpu_user_regs *regs);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB)
+/**************************************************************************/
+/* Virtual TLB entries
+ *
+ * We keep a cache of virtual-to-physical translations that we have seen
+ * since the last TLB flush. This is safe to use for frame translations,
+ * but callers that use the rights need to re-check the actual guest tables
+ * before triggering a fault.
+ *
+ * Lookups and updates are protected by a per-vTLB (and hence per-vcpu)
+ * lock. This lock is held *only* while reading or writing the table,
+ * so it is safe to take in any non-interrupt context. Most lookups
+ * happen with v==current, so we expect contention to be low.
+ */
+
+#define VTLB_ENTRIES 13
+
+struct shadow_vtlb {
+ unsigned long page_number; /* Guest virtual address >> PAGE_SHIFT */
+ unsigned long frame_number; /* Guest physical address >> PAGE_SHIFT */
+ u32 flags; /* Accumulated guest pte flags, or 0 for an empty slot. */
+};
+
+/* Call whenever the guest flushes hit actual TLB */
+static inline void vtlb_flush(struct vcpu *v)
+{
+ spin_lock(&v->arch.paging.vtlb_lock);
+ memset(v->arch.paging.vtlb, 0, VTLB_ENTRIES * sizeof (struct shadow_vtlb));
+ spin_unlock(&v->arch.paging.vtlb_lock);
+}
+
+static inline int vtlb_hash(unsigned long page_number)
+{
+ return page_number % VTLB_ENTRIES;
+}
+
+/* Put a translation into the vTLB, potentially clobbering an old one */
+static inline void vtlb_insert(struct vcpu *v, struct shadow_vtlb entry)
+{
+ spin_lock(&v->arch.paging.vtlb_lock);
+ v->arch.paging.vtlb[vtlb_hash(entry.page_number)] = entry;
+ spin_unlock(&v->arch.paging.vtlb_lock);
+}
+
+/* Look a translation up in the vTLB. Returns 0 if not found. */
+static inline int vtlb_lookup(struct vcpu *v, unsigned long va,
+ struct shadow_vtlb *result)
+{
+ unsigned long page_number = va >> PAGE_SHIFT;
+ int rv = 0;
+ int i = vtlb_hash(page_number);
+
+ spin_lock(&v->arch.paging.vtlb_lock);
+ if ( v->arch.paging.vtlb[i].flags != 0
+ && v->arch.paging.vtlb[i].page_number == page_number )
+ {
+ rv = 1;
+ result[0] = v->arch.paging.vtlb[i];
+ }
+ spin_unlock(&v->arch.paging.vtlb_lock);
+ return rv;
+}
+#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) */
+
+
#endif /* _XEN_SHADOW_PRIVATE_H */
/*
diff -r 13eca4bf2c69 -r bd3d6b4c52ec xen/arch/x86/mm/shadow/types.h
--- a/xen/arch/x86/mm/shadow/types.h Fri Jun 01 14:32:11 2007 +0100
+++ b/xen/arch/x86/mm/shadow/types.h Fri Jun 01 14:50:52 2007 +0100
@@ -553,6 +553,9 @@ accumulate_guest_flags(struct vcpu *v, w
{
u32 accumulated_flags;
+ if ( unlikely(!(guest_l1e_get_flags(gw->eff_l1e) & _PAGE_PRESENT)) )
+ return 0;
+
// We accumulate the permission flags with bitwise ANDing.
// This works for the PRESENT bit, RW bit, and USER bit.
// For the NX bit, however, the polarity is wrong, so we accumulate the
diff -r 13eca4bf2c69 -r bd3d6b4c52ec xen/include/asm-x86/domain.h
--- a/xen/include/asm-x86/domain.h Fri Jun 01 14:32:11 2007 +0100
+++ b/xen/include/asm-x86/domain.h Fri Jun 01 14:50:52 2007 +0100
@@ -173,6 +173,9 @@ struct paging_vcpu {
unsigned int translate_enabled:1;
/* HVM guest: last emulate was to a pagetable */
unsigned int last_write_was_pt:1;
+ /* Translated guest: virtual TLB */
+ struct shadow_vtlb *vtlb;
+ spinlock_t vtlb_lock;
/* paging support extension */
struct shadow_vcpu shadow;
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home