Provide fast write emulation path to release shadow lock. Basically we can consider shadow fault logic into two parts, with 1st part to cover logistic work like validating guest page table or fix shadow table, and the 2nd part for write emulation. However there's one scenario we can optimize to skip the 1st part. For previous successfully emulated virtual frame, it's very likely approaching at write emulation logic again if next adjacent shadow fault is hitting same virtual frame. It's wasteful to re-walk 1st part which is already covered by last shadow fault. In this case, actually we can jump to emulation code early, without any lock acquisition until final shadow validation for write emulation. By perfc counts on 64bit SMP HVM guest, 89% of total shadow write emulation are observed falling into this fast path when doing kernel build in guest. Signed-off-by Kevin Tian diff -r 64088e89251d xen/arch/x86/mm/shadow/common.c --- a/xen/arch/x86/mm/shadow/common.c Tue Jan 15 04:49:23 2008 +0800 +++ b/xen/arch/x86/mm/shadow/common.c Wed Jan 16 05:29:43 2008 +0800 @@ -1017,13 +1017,18 @@ void shadow_free(struct domain *d, mfn_t for ( i = 0; i < 1<arch.paging.shadow.last_writeable_pte_smfn == mfn_x(smfn) + i ) v->arch.paging.shadow.last_writeable_pte_smfn = 0; +#endif +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + v->arch.paging.last_write_emul_ok = 0; +#endif } #endif /* Strip out the type: this is now a free shadow page */ diff -r 64088e89251d xen/arch/x86/mm/shadow/multi.c --- a/xen/arch/x86/mm/shadow/multi.c Tue Jan 15 04:49:23 2008 +0800 +++ b/xen/arch/x86/mm/shadow/multi.c Tue Jan 22 09:41:11 2008 +0800 @@ -2594,8 +2594,8 @@ static inline void check_for_early_unsha static inline void check_for_early_unshadow(struct vcpu *v, mfn_t gmfn) { #if SHADOW_OPTIMIZATIONS & SHOPT_EARLY_UNSHADOW - if ( v->arch.paging.shadow.last_emulated_mfn == mfn_x(gmfn) && - sh_mfn_is_a_page_table(gmfn) ) + if ( v->arch.paging.shadow.last_emulated_mfn_for_unshadow == mfn_x(gmfn) + && sh_mfn_is_a_page_table(gmfn) ) { u32 flags = mfn_to_page(gmfn)->shadow_flags; if ( !(flags & (SHF_L2_32|SHF_L2_PAE|SHF_L2H_PAE|SHF_L4_64)) ) @@ -2604,7 +2604,7 @@ static inline void check_for_early_unsha sh_remove_shadows(v, gmfn, 1, 0 /* Fast, can fail to unshadow */ ); } } - v->arch.paging.shadow.last_emulated_mfn = mfn_x(gmfn); + v->arch.paging.shadow.last_emulated_mfn_for_unshadow = mfn_x(gmfn); #endif } @@ -2612,7 +2612,7 @@ static inline void reset_early_unshadow( static inline void reset_early_unshadow(struct vcpu *v) { #if SHADOW_OPTIMIZATIONS & SHOPT_EARLY_UNSHADOW - v->arch.paging.shadow.last_emulated_mfn = INVALID_MFN; + v->arch.paging.shadow.last_emulated_mfn_for_unshadow = INVALID_MFN; #endif } @@ -2720,12 +2720,40 @@ static int sh_page_fault(struct vcpu *v, int r; fetch_type_t ft = 0; p2m_type_t p2mt; +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + int fast_emul = 0; +#endif SHADOW_PRINTK("d:v=%u:%u va=%#lx err=%u, rip=%lx\n", v->domain->domain_id, v->vcpu_id, va, regs->error_code, regs->rip); perfc_incr(shadow_fault); + +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + /* If faulting virtual frame is successfully emulated in closely last + * shadow fault, it's highly likely to reach same emulation action + * for same virtual frame. Try to emulate early which can release + * shadow lock further. + */ + if ( v->arch.paging.last_write_emul_ok + && v->arch.paging.shadow.last_emulated_vfn == (va >> PAGE_SHIFT) ) + { + /* check whether error code is 3, or else fall back to normal path + * in case of some validation is required + */ + if ( regs->error_code == (PFEC_write_access | PFEC_page_present) ) + { + fast_emul = 1; + gmfn = v->arch.paging.shadow.last_emulated_mfn; + perfc_incr(shadow_fault_fast_emulate); + goto early_emulation; + } + else + v->arch.paging.last_write_emul_ok = 0; + } +#endif + // // XXX: Need to think about eventually mapping superpages directly in the // shadow (when possible), as opposed to splintering them into a @@ -2936,6 +2964,17 @@ static int sh_page_fault(struct vcpu *v, goto done; } + /* + * We don't need to hold the lock for the whole emulation; we will + * take it again when we write to the pagetables. + */ + sh_audit_gw(v, &gw); + shadow_audit_tables(v); + shadow_unlock(d); + +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + early_emulation: +#endif if ( is_hvm_domain(d) ) { /* @@ -2947,6 +2986,13 @@ static int sh_page_fault(struct vcpu *v, */ if ( unlikely(hvm_event_pending(v)) ) { +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + if ( fast_emul ) + { + perfc_incr(shadow_fault_fast_emulate_fail); + v->arch.paging.last_write_emul_ok = 0; + } +#endif gdprintk(XENLOG_DEBUG, "write to pagetable during event " "injection: cr2=%#lx, mfn=%#lx\n", va, mfn_x(gmfn)); @@ -2957,14 +3003,6 @@ static int sh_page_fault(struct vcpu *v, SHADOW_PRINTK("emulate: eip=%#lx esp=%#lx\n", (unsigned long)regs->eip, (unsigned long)regs->esp); - - /* - * We don't need to hold the lock for the whole emulation; we will - * take it again when we write to the pagetables. - */ - sh_audit_gw(v, &gw); - shadow_audit_tables(v); - shadow_unlock(d); emul_ops = shadow_init_emulation(&emul_ctxt, regs); @@ -2979,14 +3017,43 @@ static int sh_page_fault(struct vcpu *v, */ if ( r == X86EMUL_UNHANDLEABLE ) { + perfc_incr(shadow_fault_emulate_failed); +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + if ( fast_emul ) + { + perfc_incr(shadow_fault_fast_emulate_fail); + v->arch.paging.last_write_emul_ok = 0; + } +#endif SHADOW_PRINTK("emulator failure, unshadowing mfn %#lx\n", mfn_x(gmfn)); - perfc_incr(shadow_fault_emulate_failed); /* If this is actually a page table, then we have a bug, and need * to support more operations in the emulator. More likely, * though, this is a hint that this page should not be shadowed. */ shadow_remove_all_shadows(v, gmfn); } + +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + /* Record successfully emulated information as heuristics to next + * fault on same virtual frame for acceleration. But be careful + * to verify its attribute still as page table, or else unshadow + * triggered in write emulation normally requires a re-sync with + * guest page table to recover r/w permission. Incorrect record + * for such case will cause unexpected more shadow faults due to + * propagation is skipped. + */ + if ( (r == X86EMUL_OKAY) && sh_mfn_is_a_page_table(gmfn) ) + { + if ( !fast_emul ) + { + v->arch.paging.shadow.last_emulated_vfn = va >> PAGE_SHIFT; + v->arch.paging.shadow.last_emulated_mfn = gmfn; + v->arch.paging.last_write_emul_ok = 1; + } + } + else if ( fast_emul ) + v->arch.paging.last_write_emul_ok = 0; +#endif #if GUEST_PAGING_LEVELS == 3 /* PAE guest */ if ( r == X86EMUL_OKAY ) { @@ -3057,6 +3124,10 @@ sh_invlpg(struct vcpu *v, unsigned long #if (SHADOW_OPTIMIZATIONS & SHOPT_VIRTUAL_TLB) /* No longer safe to use cached gva->gfn translations */ vtlb_flush(v); +#endif + +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + v->arch.paging.last_write_emul_ok = 0; #endif /* First check that we can safely read the shadow l2e. SMP/PAE linux can @@ -3786,6 +3857,10 @@ sh_update_cr3(struct vcpu *v, int do_loc vtlb_flush(v); #endif +#if SHADOW_OPTIMIZATIONS & SHOPT_FAST_EMULATION + v->arch.paging.last_write_emul_ok = 0; +#endif + /* Release the lock, if we took it (otherwise it's the caller's problem) */ if ( do_locking ) shadow_unlock(v->domain); } diff -r 64088e89251d xen/arch/x86/mm/shadow/private.h --- a/xen/arch/x86/mm/shadow/private.h Tue Jan 15 04:49:23 2008 +0800 +++ b/xen/arch/x86/mm/shadow/private.h Wed Jan 16 05:38:01 2008 +0800 @@ -62,8 +62,9 @@ extern int shadow_audit_enable; #define SHOPT_LINUX_L3_TOPLEVEL 0x10 /* Pin l3es on early 64bit linux */ #define SHOPT_SKIP_VERIFY 0x20 /* Skip PTE v'fy when safe to do so */ #define SHOPT_VIRTUAL_TLB 0x40 /* Cache guest v->p translations */ - -#define SHADOW_OPTIMIZATIONS 0x7f +#define SHOPT_FAST_EMULATION 0x80 /* Fast write emulation */ + +#define SHADOW_OPTIMIZATIONS 0xff /****************************************************************************** diff -r 64088e89251d xen/include/asm-x86/domain.h --- a/xen/include/asm-x86/domain.h Tue Jan 15 04:49:23 2008 +0800 +++ b/xen/include/asm-x86/domain.h Wed Jan 16 05:02:08 2008 +0800 @@ -108,10 +108,14 @@ struct shadow_vcpu { #endif /* Non-PAE guests: pointer to guest top-level pagetable */ void *guest_vtable; - /* Last MFN that we emulated a write to. */ - unsigned long last_emulated_mfn; + /* Last MFN that we emulated a write to as unshadow heuristics. */ + unsigned long last_emulated_mfn_for_unshadow; /* MFN of the last shadow that we shot a writeable mapping in */ unsigned long last_writeable_pte_smfn; + /* Last virtual frame number (VFN) that we emulated a write to. */ + unsigned long last_emulated_vfn; + /* Last MFN that we emulated a write successfully */ + unsigned long last_emulated_mfn; }; /************************************************/ @@ -189,6 +193,8 @@ struct paging_vcpu { struct paging_mode *mode; /* HVM guest: last emulate was to a pagetable */ unsigned int last_write_was_pt:1; + /* HVM guest: last write emulation succeeds */ + unsigned int last_write_emul_ok:1; /* Translated guest: virtual TLB */ struct shadow_vtlb *vtlb; spinlock_t vtlb_lock; diff -r 64088e89251d xen/include/asm-x86/perfc_defn.h --- a/xen/include/asm-x86/perfc_defn.h Tue Jan 15 04:49:23 2008 +0800 +++ b/xen/include/asm-x86/perfc_defn.h Tue Jan 22 09:38:57 2008 +0800 @@ -57,6 +57,9 @@ PERFCOUNTER(shadow_fault_emulate_failed, PERFCOUNTER(shadow_fault_emulate_failed, "shadow_fault emulator fails") PERFCOUNTER(shadow_fault_emulate_stack, "shadow_fault emulate stack write") PERFCOUNTER(shadow_fault_emulate_wp, "shadow_fault emulate for CR0.WP=0") +PERFCOUNTER(shadow_fault_fast_emulate, "shadow_fault fast emulate") +PERFCOUNTER(shadow_fault_fast_emulate_fail, + "shadow_fault fast emulate failed") PERFCOUNTER(shadow_fault_mmio, "shadow_fault handled as mmio") PERFCOUNTER(shadow_fault_fixed, "shadow_fault fixed fault") PERFCOUNTER(shadow_fault_disable_write, "shadow_fault disable write perm")