diff -r bc573e4f7319 xen/arch/x86/mm.c --- a/xen/arch/x86/mm.c Tue Jan 23 18:03:02 2007 -0600 +++ b/xen/arch/x86/mm.c Tue Jan 23 18:49:46 2007 -0600 @@ -387,7 +387,7 @@ void update_cr3(struct vcpu *v) if ( shadow_mode_enabled(v->domain) ) { - shadow_update_cr3(v); + paging_update_cr3(v); return; } @@ -2102,7 +2102,7 @@ int do_mmuext_op( case MMUEXT_INVLPG_LOCAL: if ( !shadow_mode_enabled(d) - || shadow_invlpg(v, op.arg1.linear_addr) != 0 ) + || paging_invlpg(v, op.arg1.linear_addr) != 0 ) local_flush_tlb_one(op.arg1.linear_addr); break; @@ -2745,7 +2745,7 @@ int do_update_va_mapping(unsigned long v { case UVMF_LOCAL: if ( !shadow_mode_enabled(d) - || (shadow_invlpg(current, va) != 0) ) + || (paging_invlpg(current, va) != 0) ) local_flush_tlb_one(va); break; case UVMF_ALL: diff -r bc573e4f7319 xen/arch/x86/mm/shadow/common.c --- a/xen/arch/x86/mm/shadow/common.c Tue Jan 23 18:03:02 2007 -0600 +++ b/xen/arch/x86/mm/shadow/common.c Tue Jan 23 18:56:16 2007 -0600 @@ -47,10 +47,27 @@ void shadow_domain_init(struct domain *d int i; shadow_lock_init(d); for ( i = 0; i <= SHADOW_MAX_ORDER; i++ ) - INIT_LIST_HEAD(&d->arch.shadow.freelists[i]); - INIT_LIST_HEAD(&d->arch.shadow.p2m_freelist); - INIT_LIST_HEAD(&d->arch.shadow.p2m_inuse); - INIT_LIST_HEAD(&d->arch.shadow.pinned_shadows); + INIT_LIST_HEAD(&d->arch.paging.freelists[i]); + INIT_LIST_HEAD(&d->arch.paging.p2m_freelist); + INIT_LIST_HEAD(&d->arch.paging.p2m_inuse); + INIT_LIST_HEAD(&d->arch.paging.shadow.pinned_shadows); +} + +/* Setup the shadow-specfic parts of a vcpu struct. Note: The most important + * job is to initialize the update_paging_modes() function pointer, which is + * used to initialized the rest of resources. Therefore, it really does not + * matter to have v->arch.paging.mode pointing to any mode, as long as it can + * be compiled. + */ +void shadow_vcpu_init(struct vcpu *v) +{ +#if CONFIG_PAGING_LEVELS == 4 + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); +#elif CONFIG_PAGING_LEVELS == 3 + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); +#elif CONFIG_PAGING_LEVELS == 2 + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,2,2); +#endif } @@ -265,7 +282,7 @@ hvm_emulate_write(enum x86_segment seg, if ( rc ) return rc; - return v->arch.shadow.mode->x86_emulate_write( + return v->arch.paging.mode->shadow.x86_emulate_write( v, addr, &val, bytes, sh_ctxt); } @@ -288,7 +305,7 @@ hvm_emulate_cmpxchg(enum x86_segment seg if ( rc ) return rc; - return v->arch.shadow.mode->x86_emulate_cmpxchg( + return v->arch.paging.mode->shadow.x86_emulate_cmpxchg( v, addr, old, new, bytes, sh_ctxt); } @@ -312,7 +329,7 @@ hvm_emulate_cmpxchg8b(enum x86_segment s if ( rc ) return rc; - return v->arch.shadow.mode->x86_emulate_cmpxchg8b( + return v->arch.paging.mode->shadow.x86_emulate_cmpxchg8b( v, addr, old_lo, old_hi, new_lo, new_hi, sh_ctxt); } @@ -353,7 +370,7 @@ pv_emulate_write(enum x86_segment seg, struct sh_emulate_ctxt *sh_ctxt = container_of(ctxt, struct sh_emulate_ctxt, ctxt); struct vcpu *v = current; - return v->arch.shadow.mode->x86_emulate_write( + return v->arch.paging.mode->shadow.x86_emulate_write( v, offset, &val, bytes, sh_ctxt); } @@ -368,7 +385,7 @@ pv_emulate_cmpxchg(enum x86_segment seg, struct sh_emulate_ctxt *sh_ctxt = container_of(ctxt, struct sh_emulate_ctxt, ctxt); struct vcpu *v = current; - return v->arch.shadow.mode->x86_emulate_cmpxchg( + return v->arch.paging.mode->shadow.x86_emulate_cmpxchg( v, offset, old, new, bytes, sh_ctxt); } @@ -384,7 +401,7 @@ pv_emulate_cmpxchg8b(enum x86_segment se struct sh_emulate_ctxt *sh_ctxt = container_of(ctxt, struct sh_emulate_ctxt, ctxt); struct vcpu *v = current; - return v->arch.shadow.mode->x86_emulate_cmpxchg8b( + return v->arch.paging.mode->shadow.x86_emulate_cmpxchg8b( v, offset, old_lo, old_hi, new_lo, new_hi, sh_ctxt); } @@ -721,7 +738,7 @@ static inline int chunk_is_available(str int i; for ( i = order; i <= SHADOW_MAX_ORDER; i++ ) - if ( !list_empty(&d->arch.shadow.freelists[i]) ) + if ( !list_empty(&d->arch.paging.freelists[i]) ) return 1; return 0; } @@ -783,7 +800,7 @@ void shadow_prealloc(struct domain *d, u /* Stage one: walk the list of pinned pages, unpinning them */ perfc_incrc(shadow_prealloc_1); - list_for_each_backwards_safe(l, t, &d->arch.shadow.pinned_shadows) + list_for_each_backwards_safe(l, t, &d->arch.paging.shadow.pinned_shadows) { sp = list_entry(l, struct shadow_page_info, list); smfn = shadow_page_to_mfn(sp); @@ -823,9 +840,9 @@ void shadow_prealloc(struct domain *d, u SHADOW_PRINTK("Can't pre-allocate %i shadow pages!\n" " shadow pages total = %u, free = %u, p2m=%u\n", 1 << order, - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); BUG(); } @@ -840,7 +857,7 @@ static void shadow_blow_tables(struct do int i; /* Pass one: unpin all pinned pages */ - list_for_each_backwards_safe(l,t, &d->arch.shadow.pinned_shadows) + list_for_each_backwards_safe(l,t, &d->arch.paging.shadow.pinned_shadows) { sp = list_entry(l, struct shadow_page_info, list); smfn = shadow_page_to_mfn(sp); @@ -905,9 +922,9 @@ mfn_t shadow_alloc(struct domain *d, /* Find smallest order which can satisfy the request. */ for ( i = order; i <= SHADOW_MAX_ORDER; i++ ) - if ( !list_empty(&d->arch.shadow.freelists[i]) ) - { - sp = list_entry(d->arch.shadow.freelists[i].next, + if ( !list_empty(&d->arch.paging.freelists[i]) ) + { + sp = list_entry(d->arch.paging.freelists[i].next, struct shadow_page_info, list); list_del(&sp->list); @@ -916,10 +933,10 @@ mfn_t shadow_alloc(struct domain *d, { i--; sp->order = i; - list_add_tail(&sp->list, &d->arch.shadow.freelists[i]); + list_add_tail(&sp->list, &d->arch.paging.freelists[i]); sp += 1 << i; } - d->arch.shadow.free_pages -= 1 << order; + d->arch.paging.free_pages -= 1 << order; /* Init page info fields and clear the pages */ for ( i = 0; i < 1<arch.shadow.free_pages += 1 << order; + d->arch.paging.free_pages += 1 << order; for ( i = 0; i < 1<arch.shadow.last_writeable_pte_smfn == mfn_x(smfn) + i ) - v->arch.shadow.last_writeable_pte_smfn = 0; + if ( v->arch.paging.shadow.last_writeable_pte_smfn == mfn_x(smfn) + i ) + v->arch.paging.shadow.last_writeable_pte_smfn = 0; } #endif /* Strip out the type: this is now a free shadow page */ @@ -1019,7 +1036,7 @@ void shadow_free(struct domain *d, mfn_t } sp->order = order; - list_add_tail(&sp->list, &d->arch.shadow.freelists[order]); + list_add_tail(&sp->list, &d->arch.paging.freelists[order]); } /* Divert some memory from the pool to be used by the p2m mapping. @@ -1039,13 +1056,13 @@ shadow_alloc_p2m_pages(struct domain *d) u32 i; ASSERT(shadow_locked_by_me(d)); - if ( d->arch.shadow.total_pages + if ( d->arch.paging.total_pages < (shadow_min_acceptable_pages(d) + (1<arch.shadow.p2m_pages += (1<arch.shadow.total_pages -= (1<arch.paging.p2m_pages += (1<arch.paging.total_pages -= (1<arch.shadow.p2m_freelist); + list_add_tail(&pg[i].list, &d->arch.paging.p2m_freelist); } return 1; } @@ -1069,12 +1086,12 @@ shadow_alloc_p2m_page(struct domain *d) mfn_t mfn; void *p; - if ( list_empty(&d->arch.shadow.p2m_freelist) && + if ( list_empty(&d->arch.paging.p2m_freelist) && !shadow_alloc_p2m_pages(d) ) return _mfn(0); - entry = d->arch.shadow.p2m_freelist.next; + entry = d->arch.paging.p2m_freelist.next; list_del(entry); - list_add_tail(entry, &d->arch.shadow.p2m_inuse); + list_add_tail(entry, &d->arch.paging.p2m_inuse); pg = list_entry(entry, struct page_info, list); pg->count_info = 1; mfn = page_to_mfn(pg); @@ -1188,8 +1205,8 @@ p2m_next_level(struct domain *d, mfn_t * { if ( pagetable_get_pfn(v->arch.guest_table) == pagetable_get_pfn(d->arch.phys_table) - && v->arch.shadow.mode != NULL ) - v->arch.shadow.mode->update_cr3(v, 0); + && v->arch.paging.mode != NULL ) + v->arch.paging.mode->update_cr3(v, 0); } } #endif @@ -1436,7 +1453,7 @@ static void shadow_p2m_teardown(struct d d->arch.phys_table = pagetable_null(); - list_for_each_safe(entry, n, &d->arch.shadow.p2m_inuse) + list_for_each_safe(entry, n, &d->arch.paging.p2m_inuse) { pg = list_entry(entry, struct page_info, list); list_del(entry); @@ -1451,10 +1468,10 @@ static void shadow_p2m_teardown(struct d * these pages were allocated without an owner. */ page_set_owner(pg, NULL); free_domheap_pages(pg, 0); - d->arch.shadow.p2m_pages--; + d->arch.paging.p2m_pages--; perfc_decr(shadow_alloc_count); } - list_for_each_safe(entry, n, &d->arch.shadow.p2m_freelist) + list_for_each_safe(entry, n, &d->arch.paging.p2m_freelist) { list_del(entry); pg = list_entry(entry, struct page_info, list); @@ -1462,10 +1479,10 @@ static void shadow_p2m_teardown(struct d /* Free should not decrement domain's total allocation. */ page_set_owner(pg, NULL); free_domheap_pages(pg, 0); - d->arch.shadow.p2m_pages--; + d->arch.paging.p2m_pages--; perfc_decr(shadow_alloc_count); } - ASSERT(d->arch.shadow.p2m_pages == 0); + ASSERT(d->arch.paging.p2m_pages == 0); } /* Set the pool of shadow pages to the required number of pages. @@ -1491,11 +1508,11 @@ static unsigned int sh_set_allocation(st pages = (pages + ((1<arch.shadow.total_pages, pages); - - while ( d->arch.shadow.total_pages != pages ) - { - if ( d->arch.shadow.total_pages < pages ) + d->arch.paging.total_pages, pages); + + while ( d->arch.paging.total_pages != pages ) + { + if ( d->arch.paging.total_pages < pages ) { /* Need to allocate more memory from domheap */ sp = (struct shadow_page_info *) @@ -1505,8 +1522,8 @@ static unsigned int sh_set_allocation(st SHADOW_PRINTK("failed to allocate shadow pages.\n"); return -ENOMEM; } - d->arch.shadow.free_pages += 1<arch.shadow.total_pages += 1<arch.paging.free_pages += 1<arch.paging.total_pages += 1<order = SHADOW_MAX_ORDER; list_add_tail(&sp->list, - &d->arch.shadow.freelists[SHADOW_MAX_ORDER]); + &d->arch.paging.freelists[SHADOW_MAX_ORDER]); } - else if ( d->arch.shadow.total_pages > pages ) + else if ( d->arch.paging.total_pages > pages ) { /* Need to return memory to domheap */ shadow_prealloc(d, SHADOW_MAX_ORDER); - ASSERT(!list_empty(&d->arch.shadow.freelists[SHADOW_MAX_ORDER])); - sp = list_entry(d->arch.shadow.freelists[SHADOW_MAX_ORDER].next, + ASSERT(!list_empty(&d->arch.paging.freelists[SHADOW_MAX_ORDER])); + sp = list_entry(d->arch.paging.freelists[SHADOW_MAX_ORDER].next, struct shadow_page_info, list); list_del(&sp->list); - d->arch.shadow.free_pages -= 1<arch.shadow.total_pages -= 1<arch.paging.free_pages -= 1<arch.paging.total_pages -= 1<arch.shadow.total_pages; + unsigned int pg = d->arch.paging.total_pages; return ((pg >> (20 - PAGE_SHIFT)) + ((pg & ((1 << (20 - PAGE_SHIFT)) - 1)) ? 1 : 0)); } @@ -1583,7 +1600,7 @@ static void sh_hash_audit_bucket(struct if ( !(SHADOW_AUDIT_ENABLE) ) return; - sp = d->arch.shadow.hash_table[bucket]; + sp = d->arch.paging.shadow.hash_table[bucket]; while ( sp ) { /* Not a shadow? */ @@ -1652,13 +1669,13 @@ static int shadow_hash_alloc(struct doma struct shadow_page_info **table; ASSERT(shadow_locked_by_me(d)); - ASSERT(!d->arch.shadow.hash_table); + ASSERT(!d->arch.paging.shadow.hash_table); table = xmalloc_array(struct shadow_page_info *, SHADOW_HASH_BUCKETS); if ( !table ) return 1; memset(table, 0, SHADOW_HASH_BUCKETS * sizeof (struct shadow_page_info *)); - d->arch.shadow.hash_table = table; + d->arch.paging.shadow.hash_table = table; return 0; } @@ -1667,10 +1684,10 @@ static void shadow_hash_teardown(struct static void shadow_hash_teardown(struct domain *d) { ASSERT(shadow_locked_by_me(d)); - ASSERT(d->arch.shadow.hash_table); - - xfree(d->arch.shadow.hash_table); - d->arch.shadow.hash_table = NULL; + ASSERT(d->arch.paging.shadow.hash_table); + + xfree(d->arch.paging.shadow.hash_table); + d->arch.paging.shadow.hash_table = NULL; } @@ -1683,7 +1700,7 @@ mfn_t shadow_hash_lookup(struct vcpu *v, key_t key; ASSERT(shadow_locked_by_me(d)); - ASSERT(d->arch.shadow.hash_table); + ASSERT(d->arch.paging.shadow.hash_table); ASSERT(t); sh_hash_audit(d); @@ -1692,16 +1709,16 @@ mfn_t shadow_hash_lookup(struct vcpu *v, key = sh_hash(n, t); sh_hash_audit_bucket(d, key); - sp = d->arch.shadow.hash_table[key]; + sp = d->arch.paging.shadow.hash_table[key]; prev = NULL; while(sp) { if ( sp->backpointer == n && sp->type == t ) { /* Pull-to-front if 'sp' isn't already the head item */ - if ( unlikely(sp != d->arch.shadow.hash_table[key]) ) + if ( unlikely(sp != d->arch.paging.shadow.hash_table[key]) ) { - if ( unlikely(d->arch.shadow.hash_walking != 0) ) + if ( unlikely(d->arch.paging.shadow.hash_walking != 0) ) /* Can't reorder: someone is walking the hash chains */ return shadow_page_to_mfn(sp); else @@ -1710,8 +1727,8 @@ mfn_t shadow_hash_lookup(struct vcpu *v, /* Delete sp from the list */ prev->next_shadow = sp->next_shadow; /* Re-insert it at the head of the list */ - sp->next_shadow = d->arch.shadow.hash_table[key]; - d->arch.shadow.hash_table[key] = sp; + sp->next_shadow = d->arch.paging.shadow.hash_table[key]; + d->arch.paging.shadow.hash_table[key] = sp; } } else @@ -1737,7 +1754,7 @@ void shadow_hash_insert(struct vcpu *v, key_t key; ASSERT(shadow_locked_by_me(d)); - ASSERT(d->arch.shadow.hash_table); + ASSERT(d->arch.paging.shadow.hash_table); ASSERT(t); sh_hash_audit(d); @@ -1748,8 +1765,8 @@ void shadow_hash_insert(struct vcpu *v, /* Insert this shadow at the top of the bucket */ sp = mfn_to_shadow_page(smfn); - sp->next_shadow = d->arch.shadow.hash_table[key]; - d->arch.shadow.hash_table[key] = sp; + sp->next_shadow = d->arch.paging.shadow.hash_table[key]; + d->arch.paging.shadow.hash_table[key] = sp; sh_hash_audit_bucket(d, key); } @@ -1763,7 +1780,7 @@ void shadow_hash_delete(struct vcpu *v, key_t key; ASSERT(shadow_locked_by_me(d)); - ASSERT(d->arch.shadow.hash_table); + ASSERT(d->arch.paging.shadow.hash_table); ASSERT(t); sh_hash_audit(d); @@ -1773,13 +1790,13 @@ void shadow_hash_delete(struct vcpu *v, sh_hash_audit_bucket(d, key); sp = mfn_to_shadow_page(smfn); - if ( d->arch.shadow.hash_table[key] == sp ) + if ( d->arch.paging.shadow.hash_table[key] == sp ) /* Easy case: we're deleting the head item. */ - d->arch.shadow.hash_table[key] = sp->next_shadow; + d->arch.paging.shadow.hash_table[key] = sp->next_shadow; else { /* Need to search for the one we want */ - x = d->arch.shadow.hash_table[key]; + x = d->arch.paging.shadow.hash_table[key]; while ( 1 ) { ASSERT(x); /* We can't have hit the end, since our target is @@ -1818,15 +1835,15 @@ static void hash_foreach(struct vcpu *v, /* Say we're here, to stop hash-lookups reordering the chains */ ASSERT(shadow_locked_by_me(d)); - ASSERT(d->arch.shadow.hash_walking == 0); - d->arch.shadow.hash_walking = 1; + ASSERT(d->arch.paging.shadow.hash_walking == 0); + d->arch.paging.shadow.hash_walking = 1; for ( i = 0; i < SHADOW_HASH_BUCKETS; i++ ) { /* WARNING: This is not safe against changes to the hash table. * The callback *must* return non-zero if it has inserted or * deleted anything from the hash (lookups are OK, though). */ - for ( x = d->arch.shadow.hash_table[i]; x; x = x->next_shadow ) + for ( x = d->arch.paging.shadow.hash_table[i]; x; x = x->next_shadow ) { if ( callback_mask & (1 << x->type) ) { @@ -1839,7 +1856,7 @@ static void hash_foreach(struct vcpu *v, } if ( done ) break; } - d->arch.shadow.hash_walking = 0; + d->arch.paging.shadow.hash_walking = 0; } @@ -2008,15 +2025,15 @@ int sh_remove_write_access(struct vcpu * * and that mapping is likely to be in the current pagetable, * in the guest's linear map (on non-HIGHPTE linux and windows)*/ -#define GUESS(_a, _h) do { \ - if ( v->arch.shadow.mode->guess_wrmap(v, (_a), gmfn) ) \ - perfc_incrc(shadow_writeable_h_ ## _h); \ - if ( (pg->u.inuse.type_info & PGT_count_mask) == 0 ) \ - return 1; \ +#define GUESS(_a, _h) do { \ + if ( v->arch.paging.mode->shadow.guess_wrmap(v, (_a), gmfn) ) \ + perfc_incrc(shadow_writeable_h_ ## _h); \ + if ( (pg->u.inuse.type_info & PGT_count_mask) == 0 ) \ + return 1; \ } while (0) - if ( v->arch.shadow.mode->guest_levels == 2 ) + if ( v->arch.paging.mode->guest_levels == 2 ) { if ( level == 1 ) /* 32bit non-PAE w2k3: linear map at 0xC0000000 */ @@ -2028,7 +2045,7 @@ int sh_remove_write_access(struct vcpu * } #if CONFIG_PAGING_LEVELS >= 3 - else if ( v->arch.shadow.mode->guest_levels == 3 ) + else if ( v->arch.paging.mode->guest_levels == 3 ) { /* 32bit PAE w2k3: linear map at 0xC0000000 */ switch ( level ) @@ -2042,7 +2059,7 @@ int sh_remove_write_access(struct vcpu * GUESS(0xC0000000UL + (gfn << PAGE_SHIFT), 4); } #if CONFIG_PAGING_LEVELS >= 4 - else if ( v->arch.shadow.mode->guest_levels == 4 ) + else if ( v->arch.paging.mode->guest_levels == 4 ) { /* 64bit w2k3: linear map at 0x0000070000000000 */ switch ( level ) @@ -2073,10 +2090,10 @@ int sh_remove_write_access(struct vcpu * * the writeable mapping by looking at the same MFN where the last * brute-force search succeeded. */ - if ( v->arch.shadow.last_writeable_pte_smfn != 0 ) + if ( v->arch.paging.shadow.last_writeable_pte_smfn != 0 ) { unsigned long old_count = (pg->u.inuse.type_info & PGT_count_mask); - mfn_t last_smfn = _mfn(v->arch.shadow.last_writeable_pte_smfn); + mfn_t last_smfn = _mfn(v->arch.paging.shadow.last_writeable_pte_smfn); int shtype = mfn_to_shadow_page(last_smfn)->type; if ( callbacks[shtype] ) @@ -2431,7 +2448,7 @@ static void sh_update_paging_modes(struc static void sh_update_paging_modes(struct vcpu *v) { struct domain *d = v->domain; - struct shadow_paging_mode *old_mode = v->arch.shadow.mode; + struct paging_mode *old_mode = v->arch.paging.mode; mfn_t old_guest_table; ASSERT(shadow_locked_by_me(d)); @@ -2446,8 +2463,8 @@ static void sh_update_paging_modes(struc // First, tear down any old shadow tables held by this vcpu. // - if ( v->arch.shadow.mode ) - v->arch.shadow.mode->detach_old_tables(v); + if ( v->arch.paging.mode ) + v->arch.paging.mode->shadow.detach_old_tables(v); if ( !is_hvm_domain(d) ) { @@ -2456,17 +2473,17 @@ static void sh_update_paging_modes(struc /// #if CONFIG_PAGING_LEVELS == 4 if ( pv_32bit_guest(v) ) - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); else - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,4,4); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,4,4); #elif CONFIG_PAGING_LEVELS == 3 - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); #elif CONFIG_PAGING_LEVELS == 2 - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,2,2); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,2,2); #else #error unexpected paging mode #endif - v->arch.shadow.translate_enabled = !!shadow_mode_translate(d); + v->arch.paging.translate_enabled = !!shadow_mode_translate(d); } else { @@ -2476,8 +2493,8 @@ static void sh_update_paging_modes(struc ASSERT(shadow_mode_translate(d)); ASSERT(shadow_mode_external(d)); - v->arch.shadow.translate_enabled = !!hvm_paging_enabled(v); - if ( !v->arch.shadow.translate_enabled ) + v->arch.paging.translate_enabled = !!hvm_paging_enabled(v); + if ( !v->arch.paging.translate_enabled ) { /* Set v->arch.guest_table to use the p2m map, and choose * the appropriate shadow mode */ @@ -2485,11 +2502,11 @@ static void sh_update_paging_modes(struc #if CONFIG_PAGING_LEVELS == 2 v->arch.guest_table = pagetable_from_pfn(pagetable_get_pfn(d->arch.phys_table)); - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,2,2); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,2,2); #elif CONFIG_PAGING_LEVELS == 3 v->arch.guest_table = pagetable_from_pfn(pagetable_get_pfn(d->arch.phys_table)); - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); #else /* CONFIG_PAGING_LEVELS == 4 */ { l4_pgentry_t *l4e; @@ -2501,7 +2518,7 @@ static void sh_update_paging_modes(struc pagetable_from_pfn(l4e_get_pfn(l4e[0])); sh_unmap_domain_page(l4e); } - v->arch.shadow.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode,3,3); #endif /* Fix up refcounts on guest_table */ get_page(mfn_to_page(pagetable_get_mfn(v->arch.guest_table)), d); @@ -2514,7 +2531,7 @@ static void sh_update_paging_modes(struc if ( hvm_long_mode_enabled(v) ) { // long mode guest... - v->arch.shadow.mode = + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode, 4, 4); } else @@ -2523,7 +2540,7 @@ static void sh_update_paging_modes(struc { #if CONFIG_PAGING_LEVELS >= 3 // 32-bit PAE mode guest... - v->arch.shadow.mode = + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode, 3, 3); #else SHADOW_ERROR("PAE not supported in 32-bit Xen\n"); @@ -2535,10 +2552,10 @@ static void sh_update_paging_modes(struc { // 32-bit 2 level guest... #if CONFIG_PAGING_LEVELS >= 3 - v->arch.shadow.mode = + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode, 3, 2); #else - v->arch.shadow.mode = + v->arch.paging.mode = &SHADOW_INTERNAL_NAME(sh_paging_mode, 2, 2); #endif } @@ -2546,25 +2563,25 @@ static void sh_update_paging_modes(struc if ( pagetable_is_null(v->arch.monitor_table) ) { - mfn_t mmfn = v->arch.shadow.mode->make_monitor_table(v); + mfn_t mmfn = v->arch.paging.mode->shadow.make_monitor_table(v); v->arch.monitor_table = pagetable_from_mfn(mmfn); make_cr3(v, mfn_x(mmfn)); hvm_update_host_cr3(v); } - if ( v->arch.shadow.mode != old_mode ) + if ( v->arch.paging.mode != old_mode ) { SHADOW_PRINTK("new paging mode: d=%u v=%u pe=%d g=%u s=%u " "(was g=%u s=%u)\n", d->domain_id, v->vcpu_id, is_hvm_domain(d) ? !!hvm_paging_enabled(v) : 1, - v->arch.shadow.mode->guest_levels, - v->arch.shadow.mode->shadow_levels, + v->arch.paging.mode->guest_levels, + v->arch.paging.mode->shadow.shadow_levels, old_mode ? old_mode->guest_levels : 0, - old_mode ? old_mode->shadow_levels : 0); + old_mode ? old_mode->shadow.shadow_levels : 0); if ( old_mode && - (v->arch.shadow.mode->shadow_levels != - old_mode->shadow_levels) ) + (v->arch.paging.mode->shadow.shadow_levels != + old_mode->shadow.shadow_levels) ) { /* Need to make a new monitor table for the new mode */ mfn_t new_mfn, old_mfn; @@ -2584,7 +2601,7 @@ static void sh_update_paging_modes(struc old_mfn = pagetable_get_mfn(v->arch.monitor_table); v->arch.monitor_table = pagetable_null(); - new_mfn = v->arch.shadow.mode->make_monitor_table(v); + new_mfn = v->arch.paging.mode->shadow.make_monitor_table(v); v->arch.monitor_table = pagetable_from_mfn(new_mfn); SHADOW_PRINTK("new monitor table %"SH_PRI_mfn "\n", mfn_x(new_mfn)); @@ -2596,7 +2613,7 @@ static void sh_update_paging_modes(struc if ( v == current ) write_ptbase(v); hvm_update_host_cr3(v); - old_mode->destroy_monitor_table(v, old_mfn); + old_mode->shadow.destroy_monitor_table(v, old_mfn); } } @@ -2606,7 +2623,7 @@ static void sh_update_paging_modes(struc // This *does* happen, at least for CR4.PGE... } - v->arch.shadow.mode->update_cr3(v, 0); + v->arch.paging.mode->update_cr3(v, 0); } void shadow_update_paging_modes(struct vcpu *v) @@ -2626,7 +2643,7 @@ static void sh_new_mode(struct domain *d ASSERT(shadow_locked_by_me(d)); ASSERT(d != current->domain); - d->arch.shadow.mode = new_mode; + d->arch.paging.shadow.mode = new_mode; if ( new_mode & SHM2_translate ) shadow_audit_p2m(d); for_each_vcpu(d, v) @@ -2672,7 +2689,7 @@ int shadow_enable(struct domain *d, u32 #endif /* Init the shadow memory allocation if the user hasn't done so */ - old_pages = d->arch.shadow.total_pages; + old_pages = d->arch.paging.total_pages; if ( old_pages == 0 ) if ( sh_set_allocation(d, 256, NULL) != 0 ) /* Use at least 1MB */ { @@ -2703,7 +2720,7 @@ int shadow_enable(struct domain *d, u32 #if (SHADOW_OPTIMIZATIONS & SHOPT_LINUX_L3_TOPLEVEL) /* We assume we're dealing with an older 64bit linux guest until we * see the guest use more than one l4 per vcpu. */ - d->arch.shadow.opt_flags = SHOPT_LINUX_L3_TOPLEVEL; + d->arch.paging.shadow.opt_flags = SHOPT_LINUX_L3_TOPLEVEL; #endif /* Update the bits */ @@ -2733,48 +2750,48 @@ void shadow_teardown(struct domain *d) /* Release the shadow and monitor tables held by each vcpu */ for_each_vcpu(d, v) { - if ( v->arch.shadow.mode ) + if ( v->arch.paging.mode ) { - v->arch.shadow.mode->detach_old_tables(v); + v->arch.paging.mode->shadow.detach_old_tables(v); if ( shadow_mode_external(d) ) { mfn = pagetable_get_mfn(v->arch.monitor_table); if ( mfn_valid(mfn) && (mfn_x(mfn) != 0) ) - v->arch.shadow.mode->destroy_monitor_table(v, mfn); + v->arch.paging.mode->shadow.destroy_monitor_table(v, mfn); v->arch.monitor_table = pagetable_null(); } } } } - if ( d->arch.shadow.total_pages != 0 ) + if ( d->arch.paging.total_pages != 0 ) { SHADOW_PRINTK("teardown of domain %u starts." " Shadow pages total = %u, free = %u, p2m=%u\n", d->domain_id, - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); /* Destroy all the shadows and release memory to domheap */ sh_set_allocation(d, 0, NULL); /* Release the hash table back to xenheap */ - if (d->arch.shadow.hash_table) + if (d->arch.paging.shadow.hash_table) shadow_hash_teardown(d); /* Release the log-dirty bitmap of dirtied pages */ sh_free_log_dirty_bitmap(d); /* Should not have any more memory held */ SHADOW_PRINTK("teardown done." " Shadow pages total = %u, free = %u, p2m=%u\n", - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); - ASSERT(d->arch.shadow.total_pages == 0); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); + ASSERT(d->arch.paging.total_pages == 0); } /* We leave the "permanent" shadow modes enabled, but clear the * log-dirty mode bit. We don't want any more mark_dirty() * calls now that we've torn down the bitmap */ - d->arch.shadow.mode &= ~SHM2_log_dirty; + d->arch.paging.shadow.mode &= ~SHM2_log_dirty; shadow_unlock(d); } @@ -2786,26 +2803,26 @@ void shadow_final_teardown(struct domain SHADOW_PRINTK("dom %u final teardown starts." " Shadow pages total = %u, free = %u, p2m=%u\n", d->domain_id, - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); /* Double-check that the domain didn't have any shadow memory. * It is possible for a domain that never got domain_kill()ed * to get here with its shadow allocation intact. */ - if ( d->arch.shadow.total_pages != 0 ) + if ( d->arch.paging.total_pages != 0 ) shadow_teardown(d); /* It is now safe to pull down the p2m map. */ - if ( d->arch.shadow.p2m_pages != 0 ) + if ( d->arch.paging.p2m_pages != 0 ) shadow_p2m_teardown(d); SHADOW_PRINTK("dom %u final teardown done." " Shadow pages total = %u, free = %u, p2m=%u\n", d->domain_id, - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); } static int shadow_one_bit_enable(struct domain *d, u32 mode) @@ -2814,12 +2831,12 @@ static int shadow_one_bit_enable(struct ASSERT(shadow_locked_by_me(d)); /* Sanity check the call */ - if ( d == current->domain || (d->arch.shadow.mode & mode) ) + if ( d == current->domain || (d->arch.paging.shadow.mode & mode) ) { return -EINVAL; } - if ( d->arch.shadow.mode == 0 ) + if ( d->arch.paging.shadow.mode == 0 ) { /* Init the shadow memory allocation and the hash table */ if ( sh_set_allocation(d, 1, NULL) != 0 @@ -2831,7 +2848,7 @@ static int shadow_one_bit_enable(struct } /* Update the bits */ - sh_new_mode(d, d->arch.shadow.mode | mode); + sh_new_mode(d, d->arch.paging.shadow.mode | mode); return 0; } @@ -2843,26 +2860,26 @@ static int shadow_one_bit_disable(struct ASSERT(shadow_locked_by_me(d)); /* Sanity check the call */ - if ( d == current->domain || !(d->arch.shadow.mode & mode) ) + if ( d == current->domain || !(d->arch.paging.shadow.mode & mode) ) { return -EINVAL; } /* Update the bits */ - sh_new_mode(d, d->arch.shadow.mode & ~mode); - if ( d->arch.shadow.mode == 0 ) + sh_new_mode(d, d->arch.paging.shadow.mode & ~mode); + if ( d->arch.paging.shadow.mode == 0 ) { /* Get this domain off shadows */ SHADOW_PRINTK("un-shadowing of domain %u starts." " Shadow pages total = %u, free = %u, p2m=%u\n", d->domain_id, - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); for_each_vcpu(d, v) { - if ( v->arch.shadow.mode ) - v->arch.shadow.mode->detach_old_tables(v); + if ( v->arch.paging.mode ) + v->arch.paging.mode->shadow.detach_old_tables(v); #if CONFIG_PAGING_LEVELS == 4 if ( !(v->arch.flags & TF_kernel_mode) ) make_cr3(v, pagetable_get_pfn(v->arch.guest_table_user)); @@ -2885,9 +2902,9 @@ static int shadow_one_bit_disable(struct SHADOW_PRINTK("un-shadowing of domain %u done." " Shadow pages total = %u, free = %u, p2m=%u\n", d->domain_id, - d->arch.shadow.total_pages, - d->arch.shadow.free_pages, - d->arch.shadow.p2m_pages); + d->arch.paging.total_pages, + d->arch.paging.free_pages, + d->arch.paging.p2m_pages); } return 0; @@ -2933,19 +2950,19 @@ static int static int sh_alloc_log_dirty_bitmap(struct domain *d) { - ASSERT(d->arch.shadow.dirty_bitmap == NULL); - d->arch.shadow.dirty_bitmap_size = + ASSERT(d->arch.paging.shadow.dirty_bitmap == NULL); + d->arch.paging.shadow.dirty_bitmap_size = (arch_get_max_pfn(d) + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1); - d->arch.shadow.dirty_bitmap = + d->arch.paging.shadow.dirty_bitmap = xmalloc_array(unsigned long, - d->arch.shadow.dirty_bitmap_size / BITS_PER_LONG); - if ( d->arch.shadow.dirty_bitmap == NULL ) - { - d->arch.shadow.dirty_bitmap_size = 0; + d->arch.paging.shadow.dirty_bitmap_size / BITS_PER_LONG); + if ( d->arch.paging.shadow.dirty_bitmap == NULL ) + { + d->arch.paging.shadow.dirty_bitmap_size = 0; return -ENOMEM; } - memset(d->arch.shadow.dirty_bitmap, 0, d->arch.shadow.dirty_bitmap_size/8); + memset(d->arch.paging.shadow.dirty_bitmap, 0, d->arch.paging.shadow.dirty_bitmap_size/8); return 0; } @@ -2953,11 +2970,11 @@ static void static void sh_free_log_dirty_bitmap(struct domain *d) { - d->arch.shadow.dirty_bitmap_size = 0; - if ( d->arch.shadow.dirty_bitmap ) - { - xfree(d->arch.shadow.dirty_bitmap); - d->arch.shadow.dirty_bitmap = NULL; + d->arch.paging.shadow.dirty_bitmap_size = 0; + if ( d->arch.paging.shadow.dirty_bitmap ) + { + xfree(d->arch.paging.shadow.dirty_bitmap); + d->arch.paging.shadow.dirty_bitmap = NULL; } } @@ -3151,11 +3168,11 @@ static int shadow_log_dirty_op( SHADOW_DEBUG(LOGDIRTY, "log-dirty %s: dom %u faults=%u dirty=%u\n", (clean) ? "clean" : "peek", d->domain_id, - d->arch.shadow.fault_count, - d->arch.shadow.dirty_count); - - sc->stats.fault_count = d->arch.shadow.fault_count; - sc->stats.dirty_count = d->arch.shadow.dirty_count; + d->arch.paging.shadow.fault_count, + d->arch.paging.shadow.dirty_count); + + sc->stats.fault_count = d->arch.paging.shadow.fault_count; + sc->stats.dirty_count = d->arch.paging.shadow.dirty_count; if ( clean ) { @@ -3164,22 +3181,22 @@ static int shadow_log_dirty_op( * but for now, we just unshadow everything except Xen. */ shadow_blow_tables(d); - d->arch.shadow.fault_count = 0; - d->arch.shadow.dirty_count = 0; + d->arch.paging.shadow.fault_count = 0; + d->arch.paging.shadow.dirty_count = 0; } if ( guest_handle_is_null(sc->dirty_bitmap) ) /* caller may have wanted just to clean the state or access stats. */ peek = 0; - if ( (peek || clean) && (d->arch.shadow.dirty_bitmap == NULL) ) + if ( (peek || clean) && (d->arch.paging.shadow.dirty_bitmap == NULL) ) { rv = -EINVAL; /* perhaps should be ENOMEM? */ goto out; } - if ( sc->pages > d->arch.shadow.dirty_bitmap_size ) - sc->pages = d->arch.shadow.dirty_bitmap_size; + if ( sc->pages > d->arch.paging.shadow.dirty_bitmap_size ) + sc->pages = d->arch.paging.shadow.dirty_bitmap_size; #define CHUNK (8*1024) /* Transfer and clear in 1kB chunks for L1 cache. */ for ( i = 0; i < sc->pages; i += CHUNK ) @@ -3193,7 +3210,7 @@ static int shadow_log_dirty_op( if ( copy_to_guest_offset( sc->dirty_bitmap, i/(8*sizeof(unsigned long)), - d->arch.shadow.dirty_bitmap+(i/(8*sizeof(unsigned long))), + d->arch.paging.shadow.dirty_bitmap+(i/(8*sizeof(unsigned long))), (bytes+sizeof(unsigned long)-1) / sizeof(unsigned long)) ) { rv = -EFAULT; @@ -3202,7 +3219,7 @@ static int shadow_log_dirty_op( } if ( clean ) - memset(d->arch.shadow.dirty_bitmap + (i/(8*sizeof(unsigned long))), + memset(d->arch.paging.shadow.dirty_bitmap + (i/(8*sizeof(unsigned long))), 0, bytes); } #undef CHUNK @@ -3224,7 +3241,7 @@ void sh_mark_dirty(struct domain *d, mfn if ( !shadow_mode_log_dirty(d) || !mfn_valid(gmfn) ) return; - ASSERT(d->arch.shadow.dirty_bitmap != NULL); + ASSERT(d->arch.paging.shadow.dirty_bitmap != NULL); /* We /really/ mean PFN here, even for non-translated guests. */ pfn = get_gpfn_from_mfn(mfn_x(gmfn)); @@ -3238,14 +3255,14 @@ void sh_mark_dirty(struct domain *d, mfn return; /* N.B. Can use non-atomic TAS because protected by shadow_lock. */ - if ( likely(pfn < d->arch.shadow.dirty_bitmap_size) ) + if ( likely(pfn < d->arch.paging.shadow.dirty_bitmap_size) ) { - if ( !__test_and_set_bit(pfn, d->arch.shadow.dirty_bitmap) ) + if ( !__test_and_set_bit(pfn, d->arch.paging.shadow.dirty_bitmap) ) { SHADOW_DEBUG(LOGDIRTY, "marked mfn %" SH_PRI_mfn " (pfn=%lx), dom %d\n", mfn_x(gmfn), pfn, d->domain_id); - d->arch.shadow.dirty_count++; + d->arch.paging.shadow.dirty_count++; } } else @@ -3255,7 +3272,7 @@ void sh_mark_dirty(struct domain *d, mfn "owner=%d c=%08x t=%" PRtype_info "\n", mfn_x(gmfn), pfn, - d->arch.shadow.dirty_bitmap_size, + d->arch.paging.shadow.dirty_bitmap_size, d->domain_id, (page_get_owner(mfn_to_page(gmfn)) ? page_get_owner(mfn_to_page(gmfn))->domain_id @@ -3295,7 +3312,7 @@ int shadow_domctl(struct domain *d, return rc; if ( is_hvm_domain(d) ) return -EINVAL; - if ( d->arch.shadow.mode & SHM2_enable ) + if ( d->arch.paging.shadow.mode & SHM2_enable ) if ( (rc = shadow_test_disable(d)) != 0 ) return rc; return 0; @@ -3393,7 +3410,7 @@ void shadow_audit_tables(struct vcpu *v) else { /* Audit only the current mode's tables */ - switch ( v->arch.shadow.mode->guest_levels ) + switch ( v->arch.paging.mode->guest_levels ) { case 2: mask = (SHF_L1_32|SHF_FL1_32|SHF_L2_32); break; case 3: mask = (SHF_L1_PAE|SHF_FL1_PAE|SHF_L2_PAE diff -r bc573e4f7319 xen/arch/x86/mm/shadow/multi.c --- a/xen/arch/x86/mm/shadow/multi.c Tue Jan 23 18:03:02 2007 -0600 +++ b/xen/arch/x86/mm/shadow/multi.c Tue Jan 23 18:56:16 2007 -0600 @@ -1572,7 +1572,7 @@ sh_make_shadow(struct vcpu *v, mfn_t gmf #if GUEST_PAGING_LEVELS == 4 #if (SHADOW_OPTIMIZATIONS & SHOPT_LINUX_L3_TOPLEVEL) if ( shadow_type == SH_type_l4_64_shadow && - unlikely(v->domain->arch.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL) ) + unlikely(v->domain->arch.paging.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL) ) { /* We're shadowing a new l4, but we've been assuming the guest uses * only one l4 per vcpu and context switches using an l4 entry. @@ -1584,7 +1584,7 @@ sh_make_shadow(struct vcpu *v, mfn_t gmf struct shadow_page_info *sp; struct vcpu *v2; int l4count = 0, vcpus = 0; - list_for_each(l, &v->domain->arch.shadow.pinned_shadows) + list_for_each(l, &v->domain->arch.paging.shadow.pinned_shadows) { sp = list_entry(l, struct shadow_page_info, list); if ( sp->type == SH_type_l4_64_shadow ) @@ -1595,13 +1595,13 @@ sh_make_shadow(struct vcpu *v, mfn_t gmf if ( l4count > 2 * vcpus ) { /* Unpin all the pinned l3 tables, and don't pin any more. */ - list_for_each_safe(l, t, &v->domain->arch.shadow.pinned_shadows) + list_for_each_safe(l, t, &v->domain->arch.paging.shadow.pinned_shadows) { sp = list_entry(l, struct shadow_page_info, list); if ( sp->type == SH_type_l3_64_shadow ) sh_unpin(v, shadow_page_to_mfn(sp)); } - v->domain->arch.shadow.opt_flags &= ~SHOPT_LINUX_L3_TOPLEVEL; + v->domain->arch.paging.shadow.opt_flags &= ~SHOPT_LINUX_L3_TOPLEVEL; } } #endif @@ -1851,7 +1851,7 @@ static shadow_l2e_t * shadow_get_and_cre #elif GUEST_PAGING_LEVELS == 3 /* PAE... */ /* We never demand-shadow PAE l3es: they are only created in * sh_update_cr3(). Check if the relevant sl3e is present. */ - shadow_l3e_t *sl3e = ((shadow_l3e_t *)&v->arch.shadow.l3table) + shadow_l3e_t *sl3e = ((shadow_l3e_t *)&v->arch.paging.shadow.l3table) + shadow_l3_linear_offset(gw->va); if ( !(shadow_l3e_get_flags(*sl3e) & _PAGE_PRESENT) ) return NULL; @@ -2358,7 +2358,7 @@ static int validate_gl1e(struct vcpu *v, gfn = guest_l1e_get_gfn(*new_gl1e); gmfn = vcpu_gfn_to_mfn(v, gfn); - mmio = (is_hvm_vcpu(v) && shadow_vcpu_mode_translate(v) && !mfn_valid(gmfn)); + mmio = (is_hvm_vcpu(v) && paging_vcpu_mode_translate(v) && !mfn_valid(gmfn)); l1e_propagate_from_guest(v, new_gl1e, _mfn(INVALID_MFN), gmfn, &new_sl1e, ft_prefetch, mmio); @@ -2506,7 +2506,7 @@ static inline void check_for_early_unsha static inline void check_for_early_unshadow(struct vcpu *v, mfn_t gmfn) { #if SHADOW_OPTIMIZATIONS & SHOPT_EARLY_UNSHADOW - if ( v->arch.shadow.last_emulated_mfn == mfn_x(gmfn) && + if ( v->arch.paging.shadow.last_emulated_mfn == mfn_x(gmfn) && sh_mfn_is_a_page_table(gmfn) ) { u32 flags = mfn_to_page(gmfn)->shadow_flags; @@ -2516,7 +2516,7 @@ static inline void check_for_early_unsha sh_remove_shadows(v, gmfn, 1, 0 /* Fast, can fail to unshadow */ ); } } - v->arch.shadow.last_emulated_mfn = mfn_x(gmfn); + v->arch.paging.shadow.last_emulated_mfn = mfn_x(gmfn); #endif } @@ -2524,7 +2524,7 @@ static inline void reset_early_unshadow( static inline void reset_early_unshadow(struct vcpu *v) { #if SHADOW_OPTIMIZATIONS & SHOPT_EARLY_UNSHADOW - v->arch.shadow.last_emulated_mfn = INVALID_MFN; + v->arch.paging.shadow.last_emulated_mfn = INVALID_MFN; #endif } @@ -2589,7 +2589,7 @@ static void sh_prefetch(struct vcpu *v, gfn = guest_l1e_get_gfn(gl1e); gmfn = vcpu_gfn_to_mfn(v, gfn); mmio = ( is_hvm_vcpu(v) - && shadow_vcpu_mode_translate(v) + && paging_vcpu_mode_translate(v) && mmio_space(gfn_to_paddr(gfn)) ); /* Propagate the entry. Safe to use a pointer to our local @@ -2631,6 +2631,7 @@ static int sh_page_fault(struct vcpu *v, SHADOW_PRINTK("d:v=%u:%u va=%#lx err=%u\n", v->domain->domain_id, v->vcpu_id, va, regs->error_code); + perfc_incrc(shadow_fault); // // XXX: Need to think about eventually mapping superpages directly in the // shadow (when possible), as opposed to splintering them into a @@ -2651,7 +2652,7 @@ static int sh_page_fault(struct vcpu *v, if ( sh_l1e_is_gnp(sl1e) ) { if ( likely(!is_hvm_domain(d) || - shadow_vcpu_mode_translate(v)) ) + paging_vcpu_mode_translate(v)) ) { /* Not-present in a guest PT: pass to the guest as * a not-present fault (by flipping two bits). */ @@ -2701,7 +2702,7 @@ static int sh_page_fault(struct vcpu *v, if ( unlikely(shadow_locked_by_me(d)) ) { SHADOW_ERROR("Recursive shadow fault: lock was taken by %s\n", - d->arch.shadow.locker_function); + d->arch.paging.shadow.locker_function); return 0; } @@ -2726,7 +2727,7 @@ static int sh_page_fault(struct vcpu *v, // if ( unlikely(!(guest_l1e_get_flags(gw.eff_l1e) & _PAGE_PRESENT)) ) { - if ( is_hvm_domain(d) && !shadow_vcpu_mode_translate(v) ) + if ( is_hvm_domain(d) && !paging_vcpu_mode_translate(v) ) { /* Not present in p2m map, means this is mmio */ gpa = va; @@ -2784,7 +2785,7 @@ static int sh_page_fault(struct vcpu *v, gfn = guest_l1e_get_gfn(gw.eff_l1e); gmfn = vcpu_gfn_to_mfn(v, gfn); mmio = (is_hvm_domain(d) - && shadow_vcpu_mode_translate(v) + && paging_vcpu_mode_translate(v) && mmio_space(gfn_to_paddr(gfn))); if ( !mmio && !mfn_valid(gmfn) ) @@ -2848,7 +2849,7 @@ static int sh_page_fault(struct vcpu *v, } perfc_incrc(shadow_fault_fixed); - d->arch.shadow.fault_count++; + d->arch.paging.shadow.fault_count++; reset_early_unshadow(v); done: @@ -2949,7 +2950,7 @@ sh_invlpg(struct vcpu *v, unsigned long return 0; } #elif SHADOW_PAGING_LEVELS == 3 - if ( !(l3e_get_flags(v->arch.shadow.l3table[shadow_l3_linear_offset(va)]) + if ( !(l3e_get_flags(v->arch.paging.shadow.l3table[shadow_l3_linear_offset(va)]) & _PAGE_PRESENT) ) // no need to flush anything if there's no SL2... return 0; @@ -2993,6 +2994,9 @@ sh_gva_to_gfn(struct vcpu *v, unsigned l walk_t gw; gfn_t gfn; + if ( unlikely(!paging_vcpu_mode_translate(v)) ) + return va >> PAGE_SHIFT; + guest_walk_tables(v, va, &gw, 0); gfn = guest_walk_to_gfn(&gw); unmap_walk(v, &gw); @@ -3006,7 +3010,12 @@ sh_gva_to_gpa(struct vcpu *v, unsigned l /* Called to translate a guest virtual address to what the *guest* * pagetables would map it to. */ { - unsigned long gfn = sh_gva_to_gfn(v, va); + unsigned long gfn; + + if ( unlikely(!paging_vcpu_mode_translate(v)) ) + return (paddr_t) va; + + gfn = sh_gva_to_gfn(v, va); if ( gfn == INVALID_GFN ) return 0; else @@ -3120,7 +3129,7 @@ sh_update_linear_entries(struct vcpu *v) } /* Shadow l3 tables are made up by sh_update_cr3 */ - sl3e = v->arch.shadow.l3table; + sl3e = v->arch.paging.shadow.l3table; for ( i = 0; i < SHADOW_L3_PAGETABLE_ENTRIES; i++ ) { @@ -3161,13 +3170,13 @@ sh_update_linear_entries(struct vcpu *v) #if GUEST_PAGING_LEVELS == 2 /* Shadow l3 tables were built by sh_update_cr3 */ if ( shadow_mode_external(d) ) - shadow_l3e = (shadow_l3e_t *)&v->arch.shadow.l3table; + shadow_l3e = (shadow_l3e_t *)&v->arch.paging.shadow.l3table; else BUG(); /* PV 2-on-3 is not supported yet */ #else /* GUEST_PAGING_LEVELS == 3 */ - shadow_l3e = (shadow_l3e_t *)&v->arch.shadow.l3table; + shadow_l3e = (shadow_l3e_t *)&v->arch.paging.shadow.l3table; /* Always safe to use guest_vtable, because it's globally mapped */ guest_l3e = v->arch.guest_vtable; @@ -3410,7 +3419,7 @@ sh_update_cr3(struct vcpu *v, int do_loc if ( do_locking ) shadow_lock(v->domain); ASSERT(shadow_locked_by_me(v->domain)); - ASSERT(v->arch.shadow.mode); + ASSERT(v->arch.paging.mode); //// //// vcpu->arch.guest_table is already set @@ -3425,7 +3434,7 @@ sh_update_cr3(struct vcpu *v, int do_loc ASSERT(shadow_mode_external(d)); // Is paging enabled on this vcpu? - if ( shadow_vcpu_mode_translate(v) ) + if ( paging_vcpu_mode_translate(v) ) { gfn = _gfn(paddr_to_pfn(hvm_get_guest_ctrl_reg(v, 3))); gmfn = vcpu_gfn_to_mfn(v, gfn); @@ -3472,7 +3481,7 @@ sh_update_cr3(struct vcpu *v, int do_loc sh_unmap_domain_page_global(v->arch.guest_vtable); if ( shadow_mode_external(d) ) { - if ( shadow_vcpu_mode_translate(v) ) + if ( paging_vcpu_mode_translate(v) ) /* Paging enabled: find where in the page the l3 table is */ guest_idx = guest_index((void *)hvm_get_guest_ctrl_reg(v, 3)); else @@ -3566,7 +3575,7 @@ sh_update_cr3(struct vcpu *v, int do_loc #endif /// - /// v->arch.shadow.l3table + /// v->arch.paging.shadow.l3table /// #if SHADOW_PAGING_LEVELS == 3 { @@ -3581,7 +3590,7 @@ sh_update_cr3(struct vcpu *v, int do_loc /* 3-on-3: make a PAE l3 that points at the four l2 pages */ smfn = pagetable_get_mfn(v->arch.shadow_table[i]); #endif - v->arch.shadow.l3table[i] = + v->arch.paging.shadow.l3table[i] = (mfn_x(smfn) == 0) ? shadow_l3e_empty() : shadow_l3e_from_mfn(smfn, _PAGE_PRESENT); @@ -3605,8 +3614,8 @@ sh_update_cr3(struct vcpu *v, int do_loc /* 2-on-3 or 3-on-3: Use the PAE shadow l3 table we just fabricated. * Don't use make_cr3 because (a) we know it's below 4GB, and * (b) it's not necessarily page-aligned, and make_cr3 takes a pfn */ - ASSERT(virt_to_maddr(&v->arch.shadow.l3table) <= 0xffffffe0ULL); - v->arch.cr3 = virt_to_maddr(&v->arch.shadow.l3table); + ASSERT(virt_to_maddr(&v->arch.paging.shadow.l3table) <= 0xffffffe0ULL); + v->arch.cr3 = virt_to_maddr(&v->arch.paging.shadow.l3table); #else /* 2-on-2 or 4-on-4: Just use the shadow top-level directly */ make_cr3(v, pagetable_get_pfn(v->arch.shadow_table[0])); @@ -3622,7 +3631,7 @@ sh_update_cr3(struct vcpu *v, int do_loc ASSERT(is_hvm_domain(d)); #if SHADOW_PAGING_LEVELS == 3 /* 2-on-3 or 3-on-3: Use the PAE shadow l3 table we just fabricated */ - hvm_update_guest_cr3(v, virt_to_maddr(&v->arch.shadow.l3table)); + hvm_update_guest_cr3(v, virt_to_maddr(&v->arch.paging.shadow.l3table)); #else /* 2-on-2 or 4-on-4: Just use the shadow top-level directly */ hvm_update_guest_cr3(v, pagetable_get_paddr(v->arch.shadow_table[0])); @@ -3665,7 +3674,7 @@ static int sh_guess_wrmap(struct vcpu *v if ( !(shadow_l3e_get_flags(*sl3p) & _PAGE_PRESENT) ) return 0; #elif SHADOW_PAGING_LEVELS == 3 - sl3p = ((shadow_l3e_t *) v->arch.shadow.l3table) + sl3p = ((shadow_l3e_t *) v->arch.paging.shadow.l3table) + shadow_l3_linear_offset(vaddr); if ( !(shadow_l3e_get_flags(*sl3p) & _PAGE_PRESENT) ) return 0; @@ -3709,7 +3718,7 @@ int sh_rm_write_access_from_l1(struct vc (void) shadow_set_l1e(v, sl1e, ro_sl1e, sl1mfn); #if SHADOW_OPTIMIZATIONS & SHOPT_WRITABLE_HEURISTIC /* Remember the last shadow that we shot a writeable mapping in */ - v->arch.shadow.last_writeable_pte_smfn = mfn_x(base_sl1mfn); + v->arch.paging.shadow.last_writeable_pte_smfn = mfn_x(base_sl1mfn); #endif if ( (mfn_to_page(readonly_mfn)->u.inuse.type_info & PGT_count_mask) == 0 ) @@ -4316,30 +4325,31 @@ int sh_audit_l4_table(struct vcpu *v, mf /**************************************************************************/ /* Entry points into this mode of the shadow code. * This will all be mangled by the preprocessor to uniquify everything. */ -struct shadow_paging_mode sh_paging_mode = { - .page_fault = sh_page_fault, - .invlpg = sh_invlpg, - .gva_to_gpa = sh_gva_to_gpa, - .gva_to_gfn = sh_gva_to_gfn, - .update_cr3 = sh_update_cr3, - .map_and_validate_gl1e = sh_map_and_validate_gl1e, - .map_and_validate_gl2e = sh_map_and_validate_gl2e, - .map_and_validate_gl2he = sh_map_and_validate_gl2he, - .map_and_validate_gl3e = sh_map_and_validate_gl3e, - .map_and_validate_gl4e = sh_map_and_validate_gl4e, - .detach_old_tables = sh_detach_old_tables, - .x86_emulate_write = sh_x86_emulate_write, - .x86_emulate_cmpxchg = sh_x86_emulate_cmpxchg, - .x86_emulate_cmpxchg8b = sh_x86_emulate_cmpxchg8b, - .make_monitor_table = sh_make_monitor_table, - .destroy_monitor_table = sh_destroy_monitor_table, - .guest_map_l1e = sh_guest_map_l1e, - .guest_get_eff_l1e = sh_guest_get_eff_l1e, +struct paging_mode sh_paging_mode = { + .page_fault = sh_page_fault, + .invlpg = sh_invlpg, + .gva_to_gpa = sh_gva_to_gpa, + .gva_to_gfn = sh_gva_to_gfn, + .update_cr3 = sh_update_cr3, + .update_paging_modes = shadow_update_paging_modes, + .guest_levels = GUEST_PAGING_LEVELS, + .shadow.map_and_validate_gl1e = sh_map_and_validate_gl1e, + .shadow.map_and_validate_gl2e = sh_map_and_validate_gl2e, + .shadow.map_and_validate_gl2he = sh_map_and_validate_gl2he, + .shadow.map_and_validate_gl3e = sh_map_and_validate_gl3e, + .shadow.map_and_validate_gl4e = sh_map_and_validate_gl4e, + .shadow.detach_old_tables = sh_detach_old_tables, + .shadow.x86_emulate_write = sh_x86_emulate_write, + .shadow.x86_emulate_cmpxchg = sh_x86_emulate_cmpxchg, + .shadow.x86_emulate_cmpxchg8b = sh_x86_emulate_cmpxchg8b, + .shadow.make_monitor_table = sh_make_monitor_table, + .shadow.destroy_monitor_table = sh_destroy_monitor_table, + .shadow.guest_map_l1e = sh_guest_map_l1e, + .shadow.guest_get_eff_l1e = sh_guest_get_eff_l1e, #if SHADOW_OPTIMIZATIONS & SHOPT_WRITABLE_HEURISTIC - .guess_wrmap = sh_guess_wrmap, -#endif - .guest_levels = GUEST_PAGING_LEVELS, - .shadow_levels = SHADOW_PAGING_LEVELS, + .shadow.guess_wrmap = sh_guess_wrmap, +#endif + .shadow.shadow_levels = SHADOW_PAGING_LEVELS, }; /* diff -r bc573e4f7319 xen/arch/x86/mm/shadow/multi.h --- a/xen/arch/x86/mm/shadow/multi.h Tue Jan 23 18:03:02 2007 -0600 +++ b/xen/arch/x86/mm/shadow/multi.h Tue Jan 23 18:48:11 2007 -0600 @@ -115,5 +115,5 @@ SHADOW_INTERNAL_NAME(sh_destroy_monitor_ (struct vcpu *v, mfn_t mmfn); #endif -extern struct shadow_paging_mode +extern struct paging_mode SHADOW_INTERNAL_NAME(sh_paging_mode, SHADOW_LEVELS, GUEST_LEVELS); diff -r bc573e4f7319 xen/arch/x86/mm/shadow/private.h --- a/xen/arch/x86/mm/shadow/private.h Tue Jan 23 18:03:02 2007 -0600 +++ b/xen/arch/x86/mm/shadow/private.h Tue Jan 23 18:47:28 2007 -0600 @@ -108,36 +108,36 @@ extern int shadow_audit_enable; #error shadow.h currently requires CONFIG_SMP #endif -#define shadow_lock_init(_d) \ - do { \ - spin_lock_init(&(_d)->arch.shadow.lock); \ - (_d)->arch.shadow.locker = -1; \ - (_d)->arch.shadow.locker_function = "nobody"; \ +#define shadow_lock_init(_d) \ + do { \ + spin_lock_init(&(_d)->arch.paging.shadow.lock); \ + (_d)->arch.paging.shadow.locker = -1; \ + (_d)->arch.paging.shadow.locker_function = "nobody"; \ } while (0) #define shadow_locked_by_me(_d) \ - (current->processor == (_d)->arch.shadow.locker) - -#define shadow_lock(_d) \ - do { \ - if ( unlikely((_d)->arch.shadow.locker == current->processor) ) \ - { \ - printk("Error: shadow lock held by %s\n", \ - (_d)->arch.shadow.locker_function); \ - BUG(); \ - } \ - spin_lock(&(_d)->arch.shadow.lock); \ - ASSERT((_d)->arch.shadow.locker == -1); \ - (_d)->arch.shadow.locker = current->processor; \ - (_d)->arch.shadow.locker_function = __func__; \ + (current->processor == (_d)->arch.paging.shadow.locker) + +#define shadow_lock(_d) \ + do { \ + if ( unlikely((_d)->arch.paging.shadow.locker == current->processor) )\ + { \ + printk("Error: shadow lock held by %s\n", \ + (_d)->arch.paging.shadow.locker_function); \ + BUG(); \ + } \ + spin_lock(&(_d)->arch.paging.shadow.lock); \ + ASSERT((_d)->arch.paging.shadow.locker == -1); \ + (_d)->arch.paging.shadow.locker = current->processor; \ + (_d)->arch.paging.shadow.locker_function = __func__; \ } while (0) -#define shadow_unlock(_d) \ - do { \ - ASSERT((_d)->arch.shadow.locker == current->processor); \ - (_d)->arch.shadow.locker = -1; \ - (_d)->arch.shadow.locker_function = "nobody"; \ - spin_unlock(&(_d)->arch.shadow.lock); \ +#define shadow_unlock(_d) \ + do { \ + ASSERT((_d)->arch.paging.shadow.locker == current->processor); \ + (_d)->arch.paging.shadow.locker = -1; \ + (_d)->arch.paging.shadow.locker_function = "nobody"; \ + spin_unlock(&(_d)->arch.paging.shadow.lock); \ } while (0) @@ -304,7 +304,7 @@ static inline int sh_type_is_pinnable(st * page. When we're shadowing those kernels, we have to pin l3 * shadows so they don't just evaporate on every context switch. * For all other guests, we'd rather use the up-pointer field in l3s. */ - if ( unlikely((v->domain->arch.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL) + if ( unlikely((v->domain->arch.paging.shadow.opt_flags & SHOPT_LINUX_L3_TOPLEVEL) && CONFIG_PAGING_LEVELS >= 4 && t == SH_type_l3_64_shadow) ) return 1; @@ -502,13 +502,13 @@ sh_mfn_is_dirty(struct domain *d, mfn_t { unsigned long pfn; ASSERT(shadow_mode_log_dirty(d)); - ASSERT(d->arch.shadow.dirty_bitmap != NULL); + ASSERT(d->arch.paging.shadow.dirty_bitmap != NULL); /* We /really/ mean PFN here, even for non-translated guests. */ pfn = get_gpfn_from_mfn(mfn_x(gmfn)); if ( likely(VALID_M2P(pfn)) - && likely(pfn < d->arch.shadow.dirty_bitmap_size) - && test_bit(pfn, d->arch.shadow.dirty_bitmap) ) + && likely(pfn < d->arch.paging.shadow.dirty_bitmap_size) + && test_bit(pfn, d->arch.paging.shadow.dirty_bitmap) ) return 1; return 0; @@ -612,7 +612,7 @@ static inline int sh_pin(struct vcpu *v, sp->pinned = 1; } /* Put it at the head of the list of pinned shadows */ - list_add(&sp->list, &v->domain->arch.shadow.pinned_shadows); + list_add(&sp->list, &v->domain->arch.paging.shadow.pinned_shadows); return 1; } diff -r bc573e4f7319 xen/arch/x86/mm/shadow/types.h --- a/xen/arch/x86/mm/shadow/types.h Tue Jan 23 18:03:02 2007 -0600 +++ b/xen/arch/x86/mm/shadow/types.h Tue Jan 23 18:54:50 2007 -0600 @@ -414,7 +414,7 @@ static inline mfn_t static inline mfn_t vcpu_gfn_to_mfn(struct vcpu *v, gfn_t gfn) { - if ( !shadow_vcpu_mode_translate(v) ) + if ( !paging_vcpu_mode_translate(v) ) return _mfn(gfn_x(gfn)); return sh_gfn_to_mfn(v->domain, gfn_x(gfn)); }