Make snapshots of guest pages on unsync to allow faster revalidation of
OOS pages.
Signed-off-by: Gianluca Guida <gianluca.guida@xxxxxxxxxxxxx>
diff -r 75eda50f0d12 xen/arch/x86/mm/shadow/common.c
--- a/xen/arch/x86/mm/shadow/common.c Fri Jun 20 15:10:53 2008 +0100
+++ b/xen/arch/x86/mm/shadow/common.c Fri Jun 20 15:11:32 2008 +0100
@@ -72,7 +72,10 @@ void shadow_vcpu_init(struct vcpu *v)
int i;
for ( i = 0; i < SHADOW_OOS_PAGES; i++ )
+ {
v->arch.paging.shadow.oos[i] = _mfn(INVALID_MFN);
+ v->arch.paging.shadow.oos_snapshot[i] = _mfn(INVALID_MFN);
+ }
#endif
v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode, 3);
@@ -562,7 +565,7 @@ void oos_audit_hash_is_present(struct do
#endif
/* Update the shadow, but keep the page out of sync. */
-static inline void _sh_resync_l1(struct vcpu *v, mfn_t gmfn)
+static inline void _sh_resync_l1(struct vcpu *v, mfn_t gmfn, mfn_t snpmfn)
{
struct page_info *pg = mfn_to_page(gmfn);
@@ -571,12 +574,12 @@ static inline void _sh_resync_l1(struct
/* Call out to the appropriate per-mode resyncing function */
if ( pg->shadow_flags & SHF_L1_32 )
- SHADOW_INTERNAL_NAME(sh_resync_l1, 2)(v, gmfn);
+ SHADOW_INTERNAL_NAME(sh_resync_l1, 2)(v, gmfn, snpmfn);
else if ( pg->shadow_flags & SHF_L1_PAE )
- SHADOW_INTERNAL_NAME(sh_resync_l1, 3)(v, gmfn);
+ SHADOW_INTERNAL_NAME(sh_resync_l1, 3)(v, gmfn, snpmfn);
#if CONFIG_PAGING_LEVELS >= 4
else if ( pg->shadow_flags & SHF_L1_64 )
- SHADOW_INTERNAL_NAME(sh_resync_l1, 4)(v, gmfn);
+ SHADOW_INTERNAL_NAME(sh_resync_l1, 4)(v, gmfn, snpmfn);
#endif
}
@@ -728,7 +731,7 @@ static int oos_remove_write_access(struc
/* Pull all the entries on an out-of-sync page back into sync. */
-static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va)
+static void _sh_resync(struct vcpu *v, mfn_t gmfn, unsigned long va, mfn_t snp)
{
struct page_info *pg = mfn_to_page(gmfn);
@@ -753,7 +756,7 @@ static void _sh_resync(struct vcpu *v, m
pg->shadow_flags &= ~SHF_oos_may_write;
/* Update the shadows with current guest entries. */
- _sh_resync_l1(v, gmfn);
+ _sh_resync_l1(v, gmfn, snp);
/* Now we know all the entries are synced, and will stay that way */
pg->shadow_flags &= ~SHF_out_of_sync;
@@ -764,27 +767,41 @@ static void _sh_resync(struct vcpu *v, m
/* Add an MFN to the list of out-of-sync guest pagetables */
static void oos_hash_add(struct vcpu *v, mfn_t gmfn, unsigned long va)
{
- int idx;
+ int idx, oidx, swap = 0;
+ void *gptr, *gsnpptr;
mfn_t *oos = v->arch.paging.shadow.oos;
unsigned long *oos_va = v->arch.paging.shadow.oos_va;
+ mfn_t *oos_snapshot = v->arch.paging.shadow.oos_snapshot;
idx = mfn_x(gmfn) % SHADOW_OOS_PAGES;
+ oidx = idx;
+
if ( mfn_valid(oos[idx])
&& (mfn_x(oos[idx]) % SHADOW_OOS_PAGES) == idx )
{
/* Punt the current occupant into the next slot */
SWAP(oos[idx], gmfn);
SWAP(oos_va[idx], va);
+ swap = 1;
idx = (idx + 1) % SHADOW_OOS_PAGES;
}
if ( mfn_valid(oos[idx]) )
{
/* Crush the current occupant. */
- _sh_resync(v, oos[idx], oos_va[idx]);
+ _sh_resync(v, oos[idx], oos_va[idx], oos_snapshot[idx]);
perfc_incr(shadow_unsync_evict);
}
oos[idx] = gmfn;
oos_va[idx] = va;
+
+ if ( swap )
+ SWAP(oos_snapshot[idx], oos_snapshot[oidx]);
+
+ gptr = sh_map_domain_page(oos[oidx]);
+ gsnpptr = sh_map_domain_page(oos_snapshot[oidx]);
+ memcpy(gsnpptr, gptr, PAGE_SIZE);
+ sh_unmap_domain_page(gptr);
+ sh_unmap_domain_page(gsnpptr);
}
/* Remove an MFN from the list of out-of-sync guest pagetables */
@@ -814,25 +831,52 @@ static void oos_hash_remove(struct vcpu
BUG();
}
+mfn_t oos_snapshot_lookup(struct vcpu *v, mfn_t gmfn)
+{
+ int idx;
+ mfn_t *oos;
+ mfn_t *oos_snapshot;
+ struct domain *d = v->domain;
+
+ for_each_vcpu(d, v)
+ {
+ oos = v->arch.paging.shadow.oos;
+ oos_snapshot = v->arch.paging.shadow.oos_snapshot;
+ idx = mfn_x(gmfn) % SHADOW_OOS_PAGES;
+ if ( mfn_x(oos[idx]) != mfn_x(gmfn) )
+ idx = (idx + 1) % SHADOW_OOS_PAGES;
+ if ( mfn_x(oos[idx]) == mfn_x(gmfn) )
+ {
+ return oos_snapshot[idx];
+ }
+ }
+
+ SHADOW_ERROR("gmfn %lx was OOS but not in hash table\n", mfn_x(gmfn));
+ BUG();
+ return _mfn(INVALID_MFN);
+}
+
/* Pull a single guest page back into sync */
void sh_resync(struct vcpu *v, mfn_t gmfn)
{
int idx;
mfn_t *oos;
unsigned long *oos_va;
+ mfn_t *oos_snapshot;
struct domain *d = v->domain;
for_each_vcpu(d, v)
{
oos = v->arch.paging.shadow.oos;
oos_va = v->arch.paging.shadow.oos_va;
+ oos_snapshot = v->arch.paging.shadow.oos_snapshot;
idx = mfn_x(gmfn) % SHADOW_OOS_PAGES;
if ( mfn_x(oos[idx]) != mfn_x(gmfn) )
idx = (idx + 1) % SHADOW_OOS_PAGES;
if ( mfn_x(oos[idx]) == mfn_x(gmfn) )
{
- _sh_resync(v, gmfn, oos_va[idx]);
+ _sh_resync(v, gmfn, oos_va[idx], oos_snapshot[idx]);
oos[idx] = _mfn(INVALID_MFN);
return;
}
@@ -873,6 +917,7 @@ void sh_resync_all(struct vcpu *v, int s
struct vcpu *other;
mfn_t *oos = v->arch.paging.shadow.oos;
unsigned long *oos_va = v->arch.paging.shadow.oos_va;
+ mfn_t *oos_snapshot = v->arch.paging.shadow.oos_snapshot;
SHADOW_PRINTK("d=%d, v=%d\n", v->domain->domain_id, v->vcpu_id);
@@ -892,7 +937,7 @@ void sh_resync_all(struct vcpu *v, int s
if ( mfn_valid(oos[idx]) )
{
/* Write-protect and sync contents */
- _sh_resync(v, oos[idx], oos_va[idx]);
+ _sh_resync(v, oos[idx], oos_va[idx], oos_snapshot[idx]);
oos[idx] = _mfn(INVALID_MFN);
}
@@ -914,7 +959,7 @@ void sh_resync_all(struct vcpu *v, int s
oos = other->arch.paging.shadow.oos;
oos_va = other->arch.paging.shadow.oos_va;
-
+ oos_snapshot = other->arch.paging.shadow.oos_snapshot;
for ( idx = 0; idx < SHADOW_OOS_PAGES; idx++ )
{
if ( !mfn_valid(oos[idx]) )
@@ -925,12 +970,12 @@ void sh_resync_all(struct vcpu *v, int s
/* Update the shadows and leave the page OOS. */
if ( sh_skip_sync(v, oos[idx]) )
continue;
- _sh_resync_l1(other, oos[idx]);
+ _sh_resync_l1(other, oos[idx], oos_snapshot[idx]);
}
else
{
/* Write-protect and sync contents */
- _sh_resync(other, oos[idx], oos_va[idx]);
+ _sh_resync(other, oos[idx], oos_va[idx], oos_snapshot[idx]);
oos[idx] = _mfn(INVALID_MFN);
}
}
@@ -1233,7 +1278,8 @@ shadow_order(unsigned int shadow_type)
0, /* SH_type_l3_64_shadow */
0, /* SH_type_l4_64_shadow */
2, /* SH_type_p2m_table */
- 0 /* SH_type_monitor_table */
+ 0, /* SH_type_monitor_table */
+ 0 /* SH_type_oos_snapshot */
};
ASSERT(shadow_type < SH_type_unused);
return type_to_order[shadow_type];
@@ -2765,6 +2811,17 @@ static void sh_update_paging_modes(struc
for ( i = 0; i < SHADOW_OOS_FT_HASH * SHADOW_OOS_FT_ENTRIES; i++ )
v->arch.paging.shadow.oos_fixups[i].gmfn = _mfn(INVALID_MFN);
}
+
+ if ( mfn_x(v->arch.paging.shadow.oos_snapshot[0]) == INVALID_MFN )
+ {
+ int i;
+ for(i = 0; i < SHADOW_OOS_PAGES; i++)
+ {
+ shadow_prealloc(d, SH_type_oos_snapshot, 1);
+ v->arch.paging.shadow.oos_snapshot[i] =
+ shadow_alloc(d, SH_type_oos_snapshot, 0);
+ }
+ }
#endif /* OOS */
// Valid transitions handled by this function:
@@ -3112,6 +3169,14 @@ void shadow_teardown(struct domain *d)
free_xenheap_pages(v->arch.paging.shadow.oos_fixups,
SHADOW_OOS_FT_ORDER);
}
+
+ {
+ int i;
+ mfn_t *oos_snapshot = v->arch.paging.shadow.oos_snapshot;
+ for(i = 0; i < SHADOW_OOS_PAGES; i++)
+ if ( mfn_valid(oos_snapshot[i]) )
+ shadow_free(d, oos_snapshot[i]);
+ }
#endif /* OOS */
}
#endif /* (SHADOW_OPTIMIZATIONS & (SHOPT_VIRTUAL_TLB|SHOPT_OUT_OF_SYNC)) */
diff -r 75eda50f0d12 xen/arch/x86/mm/shadow/multi.c
--- a/xen/arch/x86/mm/shadow/multi.c Fri Jun 20 15:10:53 2008 +0100
+++ b/xen/arch/x86/mm/shadow/multi.c Fri Jun 20 15:11:32 2008 +0100
@@ -2607,6 +2607,9 @@ static int validate_gl1e(struct vcpu *v,
mfn_t gmfn;
p2m_type_t p2mt;
int result = 0;
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ mfn_t gl1mfn;
+#endif /* OOS */
perfc_incr(shadow_validate_gl1e_calls);
@@ -2614,8 +2617,25 @@ static int validate_gl1e(struct vcpu *v,
gmfn = gfn_to_mfn(v->domain, gfn, &p2mt);
l1e_propagate_from_guest(v, new_gl1e, gmfn, &new_sl1e, ft_prefetch, p2mt);
-
result |= shadow_set_l1e(v, sl1p, new_sl1e, sl1mfn);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ gl1mfn = _mfn(mfn_to_shadow_page(sl1mfn)->backpointer);
+ if ( mfn_valid(gl1mfn)
+ && mfn_is_out_of_sync(gl1mfn) )
+ {
+ /* Update the OOS snapshot. */
+ mfn_t snpmfn = oos_snapshot_lookup(v, gl1mfn);
+ guest_l1e_t *snp;
+
+ ASSERT(mfn_valid(snpmfn));
+
+ snp = sh_map_domain_page(snpmfn);
+ snp[guest_index(new_ge)] = new_gl1e;
+ sh_unmap_domain_page(snp);
+ }
+#endif /* OOS */
+
return result;
}
@@ -2626,24 +2646,44 @@ static int validate_gl1e(struct vcpu *v,
* revalidates the guest entry that corresponds to it.
* N.B. This function is called with the vcpu that unsynced the page,
* *not* the one that is causing it to be resynced. */
-void sh_resync_l1(struct vcpu *v, mfn_t gmfn)
+void sh_resync_l1(struct vcpu *v, mfn_t gl1mfn, mfn_t snpmfn)
{
mfn_t sl1mfn;
shadow_l1e_t *sl1p;
- guest_l1e_t *gl1p, *gp;
+ guest_l1e_t *gl1p, *gp, *snp;
int rc = 0;
- sl1mfn = get_shadow_status(v, gmfn, SH_type_l1_shadow);
+ ASSERT(mfn_valid(snpmfn));
+
+ sl1mfn = get_shadow_status(v, gl1mfn, SH_type_l1_shadow);
ASSERT(mfn_valid(sl1mfn)); /* Otherwise we would not have been called */
- gp = sh_map_domain_page(gmfn);
+ snp = sh_map_domain_page(snpmfn);
+ gp = sh_map_domain_page(gl1mfn);
gl1p = gp;
- SHADOW_FOREACH_L1E(sl1mfn, sl1p, &gl1p, 0, {
- rc |= validate_gl1e(v, gl1p, sl1mfn, sl1p);
+ SHADOW_FOREACH_L1E(sl1mfn, sl1p, &gl1p, 0, {
+ guest_l1e_t gl1e = *gl1p;
+ guest_l1e_t *snpl1p = (guest_l1e_t *)snp + guest_index(gl1p);
+
+ if ( memcmp(snpl1p, &gl1e, sizeof(gl1e)) )
+ {
+ gfn_t gfn;
+ mfn_t gmfn;
+ p2m_type_t p2mt;
+ shadow_l1e_t nsl1e;
+
+ gfn = guest_l1e_get_gfn(gl1e);
+ gmfn = gfn_to_mfn(v->domain, gfn, &p2mt);
+ l1e_propagate_from_guest(v, gl1e, gmfn, &nsl1e, ft_prefetch, p2mt);
+ rc |= shadow_set_l1e(v, sl1p, nsl1e, sl1mfn);
+
+ *snpl1p = gl1e;
+ }
});
sh_unmap_domain_page(gp);
+ sh_unmap_domain_page(snp);
/* Setting shadow L1 entries should never need us to flush the TLB */
ASSERT(!(rc & SHADOW_SET_FLUSH));
@@ -2891,6 +2931,10 @@ static void sh_prefetch(struct vcpu *v,
shadow_l1e_t sl1e;
u32 gflags;
p2m_type_t p2mt;
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ guest_l1e_t *snpl1p = NULL;
+#endif /* OOS */
+
/* Prefetch no further than the end of the _shadow_ l1 MFN */
dist = (PAGE_SIZE - ((unsigned long)ptr_sl1e & ~PAGE_MASK)) / sizeof sl1e;
@@ -2903,6 +2947,17 @@ static void sh_prefetch(struct vcpu *v,
/* Normal guest page; grab the next guest entry */
gl1p = sh_map_domain_page(gw->l1mfn);
gl1p += guest_l1_table_offset(gw->va);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ if ( mfn_is_out_of_sync(gw->l1mfn) )
+ {
+ mfn_t snpmfn = oos_snapshot_lookup(v, gw->l1mfn);
+
+ ASSERT(mfn_valid(snpmfn));
+ snpl1p = sh_map_domain_page(snpmfn);
+ snpl1p += guest_l1_table_offset(gw->va);
+ }
+#endif /* OOS */
}
for ( i = 1; i < dist ; i++ )
@@ -2940,9 +2995,18 @@ static void sh_prefetch(struct vcpu *v,
/* Propagate the entry. */
l1e_propagate_from_guest(v, gl1e, gmfn, &sl1e, ft_prefetch, p2mt);
(void) shadow_set_l1e(v, ptr_sl1e + i, sl1e, sl1mfn);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ if ( snpl1p != NULL )
+ snpl1p[i] = gl1e;
+#endif /* OOS */
}
if ( gl1p != NULL )
sh_unmap_domain_page(gl1p);
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ if ( snpl1p != NULL )
+ sh_unmap_domain_page(snpl1p);
+#endif /* OOS */
}
#endif /* SHADOW_OPTIMIZATIONS & SHOPT_PREFETCH */
@@ -3227,6 +3291,22 @@ static int sh_page_fault(struct vcpu *v,
/* Calculate the shadow entry and write it */
l1e_propagate_from_guest(v, gw.l1e, gmfn, &sl1e, ft, p2mt);
r = shadow_set_l1e(v, ptr_sl1e, sl1e, sl1mfn);
+
+#if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC)
+ if ( mfn_valid(gw.l1mfn)
+ && mfn_is_out_of_sync(gw.l1mfn) )
+ {
+ /* Update the OOS snapshot. */
+ mfn_t snpmfn = oos_snapshot_lookup(v, gw.l1mfn);
+ guest_l1e_t *snp;
+
+ ASSERT(mfn_valid(snpmfn));
+
+ snp = sh_map_domain_page(snpmfn);
+ snp[guest_l1_table_offset(va)] = gw.l1e;
+ sh_unmap_domain_page(snp);
+ }
+#endif /* OOS */
#if SHADOW_OPTIMIZATIONS & SHOPT_PREFETCH
/* Prefetch some more shadow entries */
diff -r 75eda50f0d12 xen/arch/x86/mm/shadow/multi.h
--- a/xen/arch/x86/mm/shadow/multi.h Fri Jun 20 15:10:53 2008 +0100
+++ b/xen/arch/x86/mm/shadow/multi.h Fri Jun 20 15:11:32 2008 +0100
@@ -119,7 +119,7 @@ SHADOW_INTERNAL_NAME(sh_paging_mode, GUE
#if SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC
extern void
SHADOW_INTERNAL_NAME(sh_resync_l1, GUEST_LEVELS)
- (struct vcpu *v, mfn_t gmfn);
+ (struct vcpu *v, mfn_t gmfn, mfn_t snpmfn);
extern int
SHADOW_INTERNAL_NAME(sh_safe_not_to_sync, GUEST_LEVELS)
diff -r 75eda50f0d12 xen/arch/x86/mm/shadow/private.h
--- a/xen/arch/x86/mm/shadow/private.h Fri Jun 20 15:10:53 2008 +0100
+++ b/xen/arch/x86/mm/shadow/private.h Fri Jun 20 15:11:32 2008 +0100
@@ -196,9 +196,9 @@ struct shadow_page_info
u32 tlbflush_timestamp;
};
struct {
- unsigned int type:4; /* What kind of shadow is this? */
+ unsigned int type:5; /* What kind of shadow is this? */
unsigned int pinned:1; /* Is the shadow pinned? */
- unsigned int count:27; /* Reference count */
+ unsigned int count:26; /* Reference count */
u32 mbz; /* Must be zero: this is where the owner
* field lives in a non-shadow page */
} __attribute__((packed));
@@ -243,7 +243,8 @@ static inline void shadow_check_page_str
#define SH_type_max_shadow (13U)
#define SH_type_p2m_table (14U) /* in use as the p2m table */
#define SH_type_monitor_table (15U) /* in use as a monitor table */
-#define SH_type_unused (16U)
+#define SH_type_oos_snapshot (16U) /* in use as OOS snapshot */
+#define SH_type_unused (17U)
/*
* What counts as a pinnable shadow?
@@ -466,6 +467,8 @@ shadow_sync_other_vcpus(struct vcpu *v,
}
void oos_audit_hash_is_present(struct domain *d, mfn_t gmfn);
+mfn_t oos_snapshot_lookup(struct vcpu *v, mfn_t gmfn);
+
#endif /* (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC) */
/******************************************************************************
diff -r 75eda50f0d12 xen/include/asm-x86/domain.h
--- a/xen/include/asm-x86/domain.h Fri Jun 20 15:10:53 2008 +0100
+++ b/xen/include/asm-x86/domain.h Fri Jun 20 15:11:32 2008 +0100
@@ -129,6 +129,7 @@ struct shadow_vcpu {
/* Shadow out-of-sync: pages that this vcpu has let go out of sync */
mfn_t oos[SHADOW_OOS_PAGES];
unsigned long oos_va[SHADOW_OOS_PAGES];
+ mfn_t oos_snapshot[SHADOW_OOS_PAGES];
struct oos_fixup {
mfn_t gmfn;
mfn_t smfn;
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home