xen/arch/x86/cpu/mcheck/vmce.c | 9 +-
xen/arch/x86/debug.c | 17 ++-
xen/arch/x86/domain.c | 27 +++++-
xen/arch/x86/domctl.c | 15 ++-
xen/arch/x86/hvm/emulate.c | 29 ++++++-
xen/arch/x86/hvm/hvm.c | 113 ++++++++++++++++++++-------
xen/arch/x86/hvm/mtrr.c | 2 +-
xen/arch/x86/hvm/stdvga.c | 4 +-
xen/arch/x86/hvm/svm/nestedsvm.c | 13 ++-
xen/arch/x86/hvm/svm/svm.c | 12 ++-
xen/arch/x86/hvm/viridian.c | 8 +-
xen/arch/x86/hvm/vmx/vmx.c | 15 ++-
xen/arch/x86/hvm/vmx/vvmx.c | 13 ++-
xen/arch/x86/mm.c | 153 +++++++++++++++++++++++++++++++-------
xen/arch/x86/mm/guest_walk.c | 31 ++++++-
xen/arch/x86/mm/hap/guest_walk.c | 15 +++-
xen/arch/x86/mm/hap/nested_hap.c | 17 +++-
xen/arch/x86/mm/mem_event.c | 23 ++++-
xen/arch/x86/mm/mem_sharing.c | 27 +++++-
xen/arch/x86/mm/p2m-pod.c | 19 +++-
xen/arch/x86/mm/p2m.c | 40 +++++----
xen/arch/x86/mm/shadow/common.c | 6 +-
xen/arch/x86/mm/shadow/multi.c | 85 +++++++++++++++------
xen/arch/x86/mm/shadow/types.h | 10 +-
xen/arch/x86/physdev.c | 8 +-
xen/arch/x86/traps.c | 19 +++-
xen/common/grant_table.c | 30 ++++++-
xen/common/memory.c | 19 ++++-
xen/common/tmem_xen.c | 21 +++-
xen/include/asm-ia64/mm.h | 2 +
xen/include/asm-x86/guest_pt.h | 6 +-
xen/include/asm-x86/p2m.h | 45 ++++++++---
32 files changed, 636 insertions(+), 217 deletions(-)
Callers of lookups into the p2m code are now variants of get_gfn. All
callers need to call put_gfn. The code behind it is a no-op at the
moment, but will change to proper locking in a later patch.
This patch does not change functionality. Only naming, and adds
put_gfn's.
set_p2m_entry retains its name because it is always called with
p2m_lock held.
This patch is humongous, unfortunately, given the dozens of call sites
involved.
After this patch, anyone using old style gfn_to_mfn will not succeed
in compiling their code. This is on purpose: adapt to the new API.
Signed-off-by: Andres Lagar-Cavilla <andres@xxxxxxxxxxxxxxxx>
Acked-by: Tim Deegan <tim@xxxxxxx>
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/cpu/mcheck/vmce.c
--- a/xen/arch/x86/cpu/mcheck/vmce.c
+++ b/xen/arch/x86/cpu/mcheck/vmce.c
@@ -574,6 +574,7 @@ int unmmap_broken_page(struct domain *d,
{
mfn_t r_mfn;
p2m_type_t pt;
+ int rc;
/* Always trust dom0's MCE handler will prevent future access */
if ( d == dom0 )
@@ -585,14 +586,16 @@ int unmmap_broken_page(struct domain *d,
if ( !is_hvm_domain(d) || !paging_mode_hap(d) )
return -ENOSYS;
- r_mfn = gfn_to_mfn_query(d, gfn, &pt);
+ rc = -1;
+ r_mfn = get_gfn_query(d, gfn, &pt);
if ( p2m_to_mask(pt) & P2M_UNMAP_TYPES)
{
ASSERT(mfn_x(r_mfn) == mfn_x(mfn));
p2m_change_type(d, gfn, pt, p2m_ram_broken);
- return 0;
+ rc = 0;
}
+ put_gfn(d, gfn);
- return -1;
+ return rc;
}
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/debug.c
--- a/xen/arch/x86/debug.c
+++ b/xen/arch/x86/debug.c
@@ -43,22 +43,23 @@
/* Returns: mfn for the given (hvm guest) vaddr */
static unsigned long
-dbg_hvm_va2mfn(dbgva_t vaddr, struct domain *dp, int toaddr)
+dbg_hvm_va2mfn(dbgva_t vaddr, struct domain *dp, int toaddr,
+ unsigned long *gfn)
{
- unsigned long mfn, gfn;
+ unsigned long mfn;
uint32_t pfec = PFEC_page_present;
p2m_type_t gfntype;
DBGP2("vaddr:%lx domid:%d\n", vaddr, dp->domain_id);
- gfn = paging_gva_to_gfn(dp->vcpu[0], vaddr, &pfec);
- if ( gfn == INVALID_GFN )
+ *gfn = paging_gva_to_gfn(dp->vcpu[0], vaddr, &pfec);
+ if ( *gfn == INVALID_GFN )
{
DBGP2("kdb:bad gfn from gva_to_gfn\n");
return INVALID_MFN;
}
- mfn = mfn_x(gfn_to_mfn(dp, gfn, &gfntype));
+ mfn = mfn_x(get_gfn(dp, *gfn, &gfntype));
if ( p2m_is_readonly(gfntype) && toaddr )
{
DBGP2("kdb:p2m_is_readonly: gfntype:%x\n", gfntype);
@@ -193,12 +194,12 @@ dbg_rw_guest_mem(dbgva_t addr, dbgbyte_t
while ( len > 0 )
{
char *va;
- unsigned long mfn, pagecnt;
+ unsigned long mfn, gfn = INVALID_GFN, pagecnt;
pagecnt = min_t(long, PAGE_SIZE - (addr & ~PAGE_MASK), len);
mfn = (dp->is_hvm
- ? dbg_hvm_va2mfn(addr, dp, toaddr)
+ ? dbg_hvm_va2mfn(addr, dp, toaddr, &gfn)
: dbg_pv_va2mfn(addr, dp, pgd3));
if ( mfn == INVALID_MFN )
break;
@@ -217,6 +218,8 @@ dbg_rw_guest_mem(dbgva_t addr, dbgbyte_t
}
unmap_domain_page(va);
+ if ( gfn != INVALID_GFN )
+ put_gfn(dp, gfn);
addr += pagecnt;
buf += pagecnt;
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/domain.c
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -720,6 +720,7 @@ int arch_set_info_guest(
struct vcpu *v, vcpu_guest_context_u c)
{
struct domain *d = v->domain;
+ unsigned long cr3_gfn;
unsigned long cr3_pfn = INVALID_MFN;
unsigned long flags, cr4;
unsigned int i;
@@ -931,7 +932,8 @@ int arch_set_info_guest(
if ( !compat )
{
- cr3_pfn = gmfn_to_mfn(d, xen_cr3_to_pfn(c.nat->ctrlreg[3]));
+ cr3_gfn = xen_cr3_to_pfn(c.nat->ctrlreg[3]);
+ cr3_pfn = get_gfn_untyped(d, cr3_gfn);
if ( !mfn_valid(cr3_pfn) ||
(paging_mode_refcounts(d)
@@ -939,16 +941,18 @@ int arch_set_info_guest(
: !get_page_and_type(mfn_to_page(cr3_pfn), d,
PGT_base_page_table)) )
{
+ put_gfn(d, cr3_gfn);
destroy_gdt(v);
return -EINVAL;
}
v->arch.guest_table = pagetable_from_pfn(cr3_pfn);
-
+ put_gfn(d, cr3_gfn);
#ifdef __x86_64__
if ( c.nat->ctrlreg[1] )
{
- cr3_pfn = gmfn_to_mfn(d, xen_cr3_to_pfn(c.nat->ctrlreg[1]));
+ cr3_gfn = xen_cr3_to_pfn(c.nat->ctrlreg[1]);
+ cr3_pfn = get_gfn_untyped(d, cr3_gfn);
if ( !mfn_valid(cr3_pfn) ||
(paging_mode_refcounts(d)
@@ -962,11 +966,13 @@ int arch_set_info_guest(
put_page(mfn_to_page(cr3_pfn));
else
put_page_and_type(mfn_to_page(cr3_pfn));
+ put_gfn(d, cr3_gfn);
destroy_gdt(v);
return -EINVAL;
}
v->arch.guest_table_user = pagetable_from_pfn(cr3_pfn);
+ put_gfn(d, cr3_gfn);
}
else if ( !(flags & VGCF_in_kernel) )
{
@@ -978,7 +984,8 @@ int arch_set_info_guest(
{
l4_pgentry_t *l4tab;
- cr3_pfn = gmfn_to_mfn(d, compat_cr3_to_pfn(c.cmp->ctrlreg[3]));
+ cr3_gfn = compat_cr3_to_pfn(c.cmp->ctrlreg[3]);
+ cr3_pfn = get_gfn_untyped(d, cr3_gfn);
if ( !mfn_valid(cr3_pfn) ||
(paging_mode_refcounts(d)
@@ -986,6 +993,7 @@ int arch_set_info_guest(
: !get_page_and_type(mfn_to_page(cr3_pfn), d,
PGT_l3_page_table)) )
{
+ put_gfn(d, cr3_gfn);
destroy_gdt(v);
return -EINVAL;
}
@@ -993,6 +1001,7 @@ int arch_set_info_guest(
l4tab = __va(pagetable_get_paddr(v->arch.guest_table));
*l4tab = l4e_from_pfn(
cr3_pfn, _PAGE_PRESENT|_PAGE_RW|_PAGE_USER|_PAGE_ACCESSED);
+ put_gfn(d, cr3_gfn);
#endif
}
@@ -1058,11 +1067,12 @@ unmap_vcpu_info(struct vcpu *v)
* event doesn't get missed.
*/
static int
-map_vcpu_info(struct vcpu *v, unsigned long mfn, unsigned offset)
+map_vcpu_info(struct vcpu *v, unsigned long gfn, unsigned offset)
{
struct domain *d = v->domain;
void *mapping;
vcpu_info_t *new_info;
+ unsigned long mfn;
int i;
if ( offset > (PAGE_SIZE - sizeof(vcpu_info_t)) )
@@ -1075,15 +1085,19 @@ map_vcpu_info(struct vcpu *v, unsigned l
if ( (v != current) && !test_bit(_VPF_down, &v->pause_flags) )
return -EINVAL;
- mfn = gmfn_to_mfn(d, mfn);
+ mfn = get_gfn_untyped(d, gfn);
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
+ {
+ put_gfn(d, gfn);
return -EINVAL;
+ }
mapping = map_domain_page_global(mfn);
if ( mapping == NULL )
{
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, gfn);
return -ENOMEM;
}
@@ -1113,6 +1127,7 @@ map_vcpu_info(struct vcpu *v, unsigned l
for ( i = 0; i < BITS_PER_EVTCHN_WORD(d); i++ )
set_bit(i, &vcpu_info(v, evtchn_pending_sel));
+ put_gfn(d, gfn);
return 0;
}
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/domctl.c
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -199,7 +199,7 @@ long arch_do_domctl(
for ( j = 0; j < k; j++ )
{
- unsigned long type = 0, mfn = gmfn_to_mfn(d, arr[j]);
+ unsigned long type = 0, mfn = get_gfn_untyped(d, arr[j]);
page = mfn_to_page(mfn);
@@ -235,6 +235,7 @@ long arch_do_domctl(
type = XEN_DOMCTL_PFINFO_XTAB;
arr[j] = type;
+ put_gfn(d, arr[j]);
}
if ( copy_to_guest_offset(domctl->u.getpageframeinfo3.array,
@@ -299,7 +300,8 @@ long arch_do_domctl(
for ( j = 0; j < k; j++ )
{
struct page_info *page;
- unsigned long mfn = gmfn_to_mfn(d, arr32[j]);
+ unsigned long gfn = arr32[j];
+ unsigned long mfn = get_gfn_untyped(d, gfn);
page = mfn_to_page(mfn);
@@ -310,8 +312,10 @@ long arch_do_domctl(
unlikely(is_xen_heap_mfn(mfn)) )
arr32[j] |= XEN_DOMCTL_PFINFO_XTAB;
else if ( xsm_getpageframeinfo(page) != 0 )
+ {
+ put_gfn(d, gfn);
continue;
- else if ( likely(get_page(page, d)) )
+ } else if ( likely(get_page(page, d)) )
{
unsigned long type = 0;
@@ -339,6 +343,7 @@ long arch_do_domctl(
else
arr32[j] |= XEN_DOMCTL_PFINFO_XTAB;
+ put_gfn(d, gfn);
}
if ( copy_to_guest_offset(domctl->u.getpageframeinfo2.array,
@@ -425,12 +430,13 @@ long arch_do_domctl(
break;
}
- mfn = gmfn_to_mfn(d, gmfn);
+ mfn = get_gfn_untyped(d, gmfn);
ret = -EACCES;
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
{
+ put_gfn(d, gmfn);
rcu_unlock_domain(d);
break;
}
@@ -443,6 +449,7 @@ long arch_do_domctl(
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, gmfn);
rcu_unlock_domain(d);
}
break;
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/emulate.c
--- a/xen/arch/x86/hvm/emulate.c
+++ b/xen/arch/x86/hvm/emulate.c
@@ -63,14 +63,18 @@ static int hvmemul_do_io(
int rc;
/* Check for paged out page */
- ram_mfn = gfn_to_mfn_unshare(curr->domain, ram_gfn, &p2mt);
+ ram_mfn = get_gfn_unshare(curr->domain, ram_gfn, &p2mt);
if ( p2m_is_paging(p2mt) )
{
p2m_mem_paging_populate(curr->domain, ram_gfn);
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_RETRY;
}
if ( p2m_is_shared(p2mt) )
+ {
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_RETRY;
+ }
/*
* Weird-sized accesses have undefined behaviour: we discard writes
@@ -82,6 +86,7 @@ static int hvmemul_do_io(
ASSERT(p_data != NULL); /* cannot happen with a REP prefix */
if ( dir == IOREQ_READ )
memset(p_data, ~0, size);
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_UNHANDLEABLE;
}
@@ -101,7 +106,10 @@ static int hvmemul_do_io(
paddr_t pa = vio->mmio_large_write_pa;
unsigned int bytes = vio->mmio_large_write_bytes;
if ( (addr >= pa) && ((addr + size) <= (pa + bytes)) )
+ {
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_OKAY;
+ }
}
else
{
@@ -111,6 +119,7 @@ static int hvmemul_do_io(
{
memcpy(p_data, &vio->mmio_large_read[addr - pa],
size);
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_OKAY;
}
}
@@ -123,15 +132,22 @@ static int hvmemul_do_io(
case HVMIO_completed:
vio->io_state = HVMIO_none;
if ( p_data == NULL )
+ {
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_UNHANDLEABLE;
+ }
goto finish_access;
case HVMIO_dispatched:
/* May have to wait for previous cycle of a multi-write to complete. */
if ( is_mmio && !value_is_ptr && (dir == IOREQ_WRITE) &&
(addr == (vio->mmio_large_write_pa +
vio->mmio_large_write_bytes)) )
+ {
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_RETRY;
+ }
default:
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_UNHANDLEABLE;
}
@@ -139,6 +155,7 @@ static int hvmemul_do_io(
{
gdprintk(XENLOG_WARNING, "WARNING: io already pending (%d)?\n",
p->state);
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_UNHANDLEABLE;
}
@@ -189,7 +206,10 @@ static int hvmemul_do_io(
}
if ( rc != X86EMUL_OKAY )
+ {
+ put_gfn(curr->domain, ram_gfn);
return rc;
+ }
finish_access:
if ( p_data != NULL )
@@ -223,6 +243,7 @@ static int hvmemul_do_io(
}
}
+ put_gfn(curr->domain, ram_gfn);
return X86EMUL_OKAY;
}
@@ -671,12 +692,14 @@ static int hvmemul_rep_movs(
if ( rc != X86EMUL_OKAY )
return rc;
- (void)gfn_to_mfn(current->domain, sgpa >> PAGE_SHIFT, &p2mt);
+ /* Unlocked works here because we get_gfn for real in whatever
+ * we call later. */
+ (void)get_gfn_unlocked(current->domain, sgpa >> PAGE_SHIFT, &p2mt);
if ( !p2m_is_ram(p2mt) && !p2m_is_grant(p2mt) )
return hvmemul_do_mmio(
sgpa, reps, bytes_per_rep, dgpa, IOREQ_READ, df, NULL);
- (void)gfn_to_mfn(current->domain, dgpa >> PAGE_SHIFT, &p2mt);
+ (void)get_gfn_unlocked(current->domain, dgpa >> PAGE_SHIFT, &p2mt);
if ( !p2m_is_ram(p2mt) && !p2m_is_grant(p2mt) )
return hvmemul_do_mmio(
dgpa, reps, bytes_per_rep, sgpa, IOREQ_WRITE, df, NULL);
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -355,26 +355,37 @@ static int hvm_set_ioreq_page(
unsigned long mfn;
void *va;
- mfn = mfn_x(gfn_to_mfn_unshare(d, gmfn, &p2mt));
+ mfn = mfn_x(get_gfn_unshare(d, gmfn, &p2mt));
if ( !p2m_is_ram(p2mt) )
+ {
+ put_gfn(d, gmfn);
return -EINVAL;
+ }
if ( p2m_is_paging(p2mt) )
{
p2m_mem_paging_populate(d, gmfn);
+ put_gfn(d, gmfn);
return -ENOENT;
}
if ( p2m_is_shared(p2mt) )
+ {
+ put_gfn(d, gmfn);
return -ENOENT;
+ }
ASSERT(mfn_valid(mfn));
page = mfn_to_page(mfn);
if ( !get_page_and_type(page, d, PGT_writable_page) )
+ {
+ put_gfn(d, gmfn);
return -EINVAL;
+ }
va = map_domain_page_global(mfn);
if ( va == NULL )
{
put_page_and_type(page);
+ put_gfn(d, gmfn);
return -ENOMEM;
}
@@ -385,6 +396,7 @@ static int hvm_set_ioreq_page(
spin_unlock(&iorp->lock);
unmap_domain_page_global(va);
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, gmfn);
return -EINVAL;
}
@@ -392,6 +404,7 @@ static int hvm_set_ioreq_page(
iorp->page = page;
spin_unlock(&iorp->lock);
+ put_gfn(d, gmfn);
domain_unpause(d);
@@ -1182,6 +1195,7 @@ int hvm_hap_nested_page_fault(unsigned l
mfn_t mfn;
struct vcpu *v = current;
struct p2m_domain *p2m;
+ int rc;
/* On Nested Virtualization, walk the guest page table.
* If this succeeds, all is fine.
@@ -1216,7 +1230,7 @@ int hvm_hap_nested_page_fault(unsigned l
}
p2m = p2m_get_hostp2m(v->domain);
- mfn = gfn_to_mfn_type_p2m(p2m, gfn, &p2mt, &p2ma, p2m_guest, NULL);
+ mfn = get_gfn_type_access(p2m, gfn, &p2mt, &p2ma, p2m_guest, NULL);
/* Check access permissions first, then handle faults */
if ( access_valid && (mfn_x(mfn) != INVALID_MFN) )
@@ -1255,8 +1269,8 @@ int hvm_hap_nested_page_fault(unsigned l
if ( violation )
{
p2m_mem_access_check(gpa, gla_valid, gla, access_r, access_w,
access_x);
-
- return 1;
+ rc = 1;
+ goto out_put_gfn;
}
}
@@ -1268,7 +1282,8 @@ int hvm_hap_nested_page_fault(unsigned l
{
if ( !handle_mmio() )
hvm_inject_exception(TRAP_gp_fault, 0, 0);
- return 1;
+ rc = 1;
+ goto out_put_gfn;
}
#ifdef __x86_64__
@@ -1281,7 +1296,8 @@ int hvm_hap_nested_page_fault(unsigned l
{
ASSERT(!p2m_is_nestedp2m(p2m));
mem_sharing_unshare_page(p2m->domain, gfn, 0);
- return 1;
+ rc = 1;
+ goto out_put_gfn;
}
#endif
@@ -1295,7 +1311,8 @@ int hvm_hap_nested_page_fault(unsigned l
*/
paging_mark_dirty(v->domain, mfn_x(mfn));
p2m_change_type(v->domain, gfn, p2m_ram_logdirty, p2m_ram_rw);
- return 1;
+ rc = 1;
+ goto out_put_gfn;
}
/* Shouldn't happen: Maybe the guest was writing to a r/o grant mapping? */
@@ -1304,10 +1321,14 @@ int hvm_hap_nested_page_fault(unsigned l
gdprintk(XENLOG_WARNING,
"trying to write to read-only grant mapping\n");
hvm_inject_exception(TRAP_gp_fault, 0, 0);
- return 1;
+ rc = 1;
+ goto out_put_gfn;
}
- return 0;
+ rc = 0;
+out_put_gfn:
+ put_gfn(p2m->domain, gfn);
+ return rc;
}
int hvm_handle_xsetbv(u64 new_bv)
@@ -1530,10 +1551,11 @@ int hvm_set_cr0(unsigned long value)
{
/* The guest CR3 must be pointing to the guest physical. */
gfn = v->arch.hvm_vcpu.guest_cr[3]>>PAGE_SHIFT;
- mfn = mfn_x(gfn_to_mfn(v->domain, gfn, &p2mt));
+ mfn = mfn_x(get_gfn(v->domain, gfn, &p2mt));
if ( !p2m_is_ram(p2mt) || !mfn_valid(mfn) ||
!get_page(mfn_to_page(mfn), v->domain))
{
+ put_gfn(v->domain, gfn);
gdprintk(XENLOG_ERR, "Invalid CR3 value = %lx (mfn=%lx)\n",
v->arch.hvm_vcpu.guest_cr[3], mfn);
domain_crash(v->domain);
@@ -1545,6 +1567,7 @@ int hvm_set_cr0(unsigned long value)
HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx, mfn = %lx",
v->arch.hvm_vcpu.guest_cr[3], mfn);
+ put_gfn(v->domain, gfn);
}
}
else if ( !(value & X86_CR0_PG) && (old_value & X86_CR0_PG) )
@@ -1621,13 +1644,17 @@ int hvm_set_cr3(unsigned long value)
{
/* Shadow-mode CR3 change. Check PDBR and update refcounts. */
HVM_DBG_LOG(DBG_LEVEL_VMMU, "CR3 value = %lx", value);
- mfn = mfn_x(gfn_to_mfn(v->domain, value >> PAGE_SHIFT, &p2mt));
+ mfn = mfn_x(get_gfn(v->domain, value >> PAGE_SHIFT, &p2mt));
if ( !p2m_is_ram(p2mt) || !mfn_valid(mfn) ||
!get_page(mfn_to_page(mfn), v->domain) )
+ {
+ put_gfn(v->domain, value >> PAGE_SHIFT);
goto bad_cr3;
+ }
put_page(pagetable_get_page(v->arch.guest_table));
v->arch.guest_table = pagetable_from_pfn(mfn);
+ put_gfn(v->domain, value >> PAGE_SHIFT);
HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx", value);
}
@@ -1764,6 +1791,7 @@ int hvm_virtual_to_linear_addr(
return 0;
}
+/* We leave this function holding a lock on the p2m entry */
static void *__hvm_map_guest_frame(unsigned long gfn, bool_t writable)
{
unsigned long mfn;
@@ -1771,13 +1799,17 @@ static void *__hvm_map_guest_frame(unsig
struct domain *d = current->domain;
mfn = mfn_x(writable
- ? gfn_to_mfn_unshare(d, gfn, &p2mt)
- : gfn_to_mfn(d, gfn, &p2mt));
+ ? get_gfn_unshare(d, gfn, &p2mt)
+ : get_gfn(d, gfn, &p2mt));
if ( (p2m_is_shared(p2mt) && writable) || !p2m_is_ram(p2mt) )
+ {
+ put_gfn(d, gfn);
return NULL;
+ }
if ( p2m_is_paging(p2mt) )
{
p2m_mem_paging_populate(d, gfn);
+ put_gfn(d, gfn);
return NULL;
}
@@ -1805,9 +1837,8 @@ void hvm_unmap_guest_frame(void *p)
unmap_domain_page(p);
}
-static void *hvm_map_entry(unsigned long va)
+static void *hvm_map_entry(unsigned long va, unsigned long *gfn)
{
- unsigned long gfn;
uint32_t pfec;
char *v;
@@ -1824,11 +1855,11 @@ static void *hvm_map_entry(unsigned long
* treat it as a kernel-mode read (i.e. no access checks).
*/
pfec = PFEC_page_present;
- gfn = paging_gva_to_gfn(current, va, &pfec);
+ *gfn = paging_gva_to_gfn(current, va, &pfec);
if ( (pfec == PFEC_page_paged) || (pfec == PFEC_page_shared) )
goto fail;
- v = hvm_map_guest_frame_rw(gfn);
+ v = hvm_map_guest_frame_rw(*gfn);
if ( v == NULL )
goto fail;
@@ -1839,9 +1870,11 @@ static void *hvm_map_entry(unsigned long
return NULL;
}
-static void hvm_unmap_entry(void *p)
+static void hvm_unmap_entry(void *p, unsigned long gfn)
{
hvm_unmap_guest_frame(p);
+ if ( p && (gfn != INVALID_GFN) )
+ put_gfn(current->domain, gfn);
}
static int hvm_load_segment_selector(
@@ -1853,6 +1886,7 @@ static int hvm_load_segment_selector(
int fault_type = TRAP_invalid_tss;
struct cpu_user_regs *regs = guest_cpu_user_regs();
struct vcpu *v = current;
+ unsigned long pdesc_gfn = INVALID_GFN;
if ( regs->eflags & X86_EFLAGS_VM )
{
@@ -1886,7 +1920,7 @@ static int hvm_load_segment_selector(
if ( ((sel & 0xfff8) + 7) > desctab.limit )
goto fail;
- pdesc = hvm_map_entry(desctab.base + (sel & 0xfff8));
+ pdesc = hvm_map_entry(desctab.base + (sel & 0xfff8), &pdesc_gfn);
if ( pdesc == NULL )
goto hvm_map_fail;
@@ -1946,7 +1980,7 @@ static int hvm_load_segment_selector(
desc.b |= 0x100;
skip_accessed_flag:
- hvm_unmap_entry(pdesc);
+ hvm_unmap_entry(pdesc, pdesc_gfn);
segr.base = (((desc.b << 0) & 0xff000000u) |
((desc.b << 16) & 0x00ff0000u) |
@@ -1962,7 +1996,7 @@ static int hvm_load_segment_selector(
return 0;
unmap_and_fail:
- hvm_unmap_entry(pdesc);
+ hvm_unmap_entry(pdesc, pdesc_gfn);
fail:
hvm_inject_exception(fault_type, sel & 0xfffc, 0);
hvm_map_fail:
@@ -1977,7 +2011,7 @@ void hvm_task_switch(
struct cpu_user_regs *regs = guest_cpu_user_regs();
struct segment_register gdt, tr, prev_tr, segr;
struct desc_struct *optss_desc = NULL, *nptss_desc = NULL, tss_desc;
- unsigned long eflags;
+ unsigned long eflags, optss_gfn = INVALID_GFN, nptss_gfn = INVALID_GFN;
int exn_raised, rc;
struct {
u16 back_link,__blh;
@@ -2003,11 +2037,11 @@ void hvm_task_switch(
goto out;
}
- optss_desc = hvm_map_entry(gdt.base + (prev_tr.sel & 0xfff8));
+ optss_desc = hvm_map_entry(gdt.base + (prev_tr.sel & 0xfff8), &optss_gfn);
if ( optss_desc == NULL )
goto out;
- nptss_desc = hvm_map_entry(gdt.base + (tss_sel & 0xfff8));
+ nptss_desc = hvm_map_entry(gdt.base + (tss_sel & 0xfff8), &nptss_gfn);
if ( nptss_desc == NULL )
goto out;
@@ -2172,8 +2206,8 @@ void hvm_task_switch(
}
out:
- hvm_unmap_entry(optss_desc);
- hvm_unmap_entry(nptss_desc);
+ hvm_unmap_entry(optss_desc, optss_gfn);
+ hvm_unmap_entry(nptss_desc, nptss_gfn);
}
#define HVMCOPY_from_guest (0u<<0)
@@ -2230,19 +2264,29 @@ static enum hvm_copy_result __hvm_copy(
gfn = addr >> PAGE_SHIFT;
}
- mfn = mfn_x(gfn_to_mfn_unshare(curr->domain, gfn, &p2mt));
+ mfn = mfn_x(get_gfn_unshare(curr->domain, gfn, &p2mt));
if ( p2m_is_paging(p2mt) )
{
p2m_mem_paging_populate(curr->domain, gfn);
+ put_gfn(curr->domain, gfn);
return HVMCOPY_gfn_paged_out;
}
if ( p2m_is_shared(p2mt) )
+ {
+ put_gfn(curr->domain, gfn);
return HVMCOPY_gfn_shared;
+ }
if ( p2m_is_grant(p2mt) )
+ {
+ put_gfn(curr->domain, gfn);
return HVMCOPY_unhandleable;
+ }
if ( !p2m_is_ram(p2mt) )
+ {
+ put_gfn(curr->domain, gfn);
return HVMCOPY_bad_gfn_to_mfn;
+ }
ASSERT(mfn_valid(mfn));
p = (char *)map_domain_page(mfn) + (addr & ~PAGE_MASK);
@@ -2273,6 +2317,7 @@ static enum hvm_copy_result __hvm_copy(
addr += count;
buf += count;
todo -= count;
+ put_gfn(curr->domain, gfn);
}
return HVMCOPY_okay;
@@ -3697,11 +3742,11 @@ long do_hvm_op(unsigned long op, XEN_GUE
for ( pfn = a.first_pfn; pfn < a.first_pfn + a.nr; pfn++ )
{
p2m_type_t t;
- mfn_t mfn = gfn_to_mfn(d, pfn, &t);
+ mfn_t mfn = get_gfn(d, pfn, &t);
if ( p2m_is_paging(t) )
{
p2m_mem_paging_populate(d, pfn);
-
+ put_gfn(d, pfn);
rc = -EINVAL;
goto param_fail3;
}
@@ -3716,6 +3761,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
/* don't take a long time and don't die either */
sh_remove_shadows(d->vcpu[0], mfn, 1, 0);
}
+ put_gfn(d, pfn);
}
param_fail3:
@@ -3739,7 +3785,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
rc = -EINVAL;
if ( is_hvm_domain(d) )
{
- gfn_to_mfn_unshare(d, a.pfn, &t);
+ get_gfn_unshare_unlocked(d, a.pfn, &t);
if ( p2m_is_mmio(t) )
a.mem_type = HVMMEM_mmio_dm;
else if ( p2m_is_readonly(t) )
@@ -3792,20 +3838,23 @@ long do_hvm_op(unsigned long op, XEN_GUE
p2m_type_t t;
p2m_type_t nt;
mfn_t mfn;
- mfn = gfn_to_mfn_unshare(d, pfn, &t);
+ mfn = get_gfn_unshare(d, pfn, &t);
if ( p2m_is_paging(t) )
{
p2m_mem_paging_populate(d, pfn);
+ put_gfn(d, pfn);
rc = -EINVAL;
goto param_fail4;
}
if ( p2m_is_shared(t) )
{
+ put_gfn(d, pfn);
rc = -EINVAL;
goto param_fail4;
}
if ( p2m_is_grant(t) )
{
+ put_gfn(d, pfn);
gdprintk(XENLOG_WARNING,
"type for pfn 0x%lx changed to grant while "
"we were working?\n", pfn);
@@ -3816,6 +3865,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
nt = p2m_change_type(d, pfn, t, memtype[a.hvmmem_type]);
if ( nt != t )
{
+ put_gfn(d, pfn);
gdprintk(XENLOG_WARNING,
"type of pfn 0x%lx changed from %d to %d while "
"we were trying to change it to %d\n",
@@ -3823,6 +3873,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
goto param_fail4;
}
}
+ put_gfn(d, pfn);
}
rc = 0;
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/mtrr.c
--- a/xen/arch/x86/hvm/mtrr.c
+++ b/xen/arch/x86/hvm/mtrr.c
@@ -389,7 +389,7 @@ uint32_t get_pat_flags(struct vcpu *v,
{
struct domain *d = v->domain;
p2m_type_t p2mt;
- gfn_to_mfn_query(d, paddr_to_pfn(gpaddr), &p2mt);
+ get_gfn_query_unlocked(d, paddr_to_pfn(gpaddr), &p2mt);
if (p2m_is_ram(p2mt))
gdprintk(XENLOG_WARNING,
"Conflict occurs for a given guest l1e flags:%x "
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/stdvga.c
--- a/xen/arch/x86/hvm/stdvga.c
+++ b/xen/arch/x86/hvm/stdvga.c
@@ -482,7 +482,7 @@ static int mmio_move(struct hvm_hw_stdvg
if ( hvm_copy_to_guest_phys(data, &tmp, p->size) !=
HVMCOPY_okay )
{
- (void)gfn_to_mfn(d, data >> PAGE_SHIFT, &p2mt);
+ (void)get_gfn_unlocked(d, data >> PAGE_SHIFT, &p2mt);
/*
* The only case we handle is vga_mem <-> vga_mem.
* Anything else disables caching and leaves it to qemu-dm.
@@ -504,7 +504,7 @@ static int mmio_move(struct hvm_hw_stdvg
if ( hvm_copy_from_guest_phys(&tmp, data, p->size) !=
HVMCOPY_okay )
{
- (void)gfn_to_mfn(d, data >> PAGE_SHIFT, &p2mt);
+ (void)get_gfn_unlocked(d, data >> PAGE_SHIFT, &p2mt);
if ( (p2mt != p2m_mmio_dm) || (data < VGA_MEM_BASE) ||
((data + p->size) > (VGA_MEM_BASE + VGA_MEM_SIZE)) )
return 0;
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/svm/nestedsvm.c
--- a/xen/arch/x86/hvm/svm/nestedsvm.c
+++ b/xen/arch/x86/hvm/svm/nestedsvm.c
@@ -81,6 +81,10 @@ int nestedsvm_vmcb_map(struct vcpu *v, u
if (nv->nv_vvmcx == NULL)
return 0;
nv->nv_vvmcxaddr = vmcbaddr;
+ /* put_gfn here even though the map survives beyond this caller.
+ * The map can likely survive beyond a hypervisor exit, thus we
+ * need to put the gfn */
+ put_gfn(current->domain, vmcbaddr >> PAGE_SHIFT);
}
return 1;
@@ -354,6 +358,7 @@ static int nsvm_vmrun_permissionmap(stru
ioport_80 = test_bit(0x80, ns_viomap);
ioport_ed = test_bit(0xed, ns_viomap);
hvm_unmap_guest_frame(ns_viomap);
+ put_gfn(current->domain, svm->ns_iomap_pa >> PAGE_SHIFT);
svm->ns_iomap = nestedhvm_vcpu_iomap_get(ioport_80, ioport_ed);
@@ -857,23 +862,25 @@ nsvm_vmcb_guest_intercepts_ioio(paddr_t
ioio_info_t ioinfo;
uint16_t port;
bool_t enabled;
+ unsigned long gfn = 0; /* gcc ... */
ioinfo.bytes = exitinfo1;
port = ioinfo.fields.port;
switch (port) {
case 0 ... 32767: /* first 4KB page */
- io_bitmap = hvm_map_guest_frame_ro(iopm_gfn);
+ gfn = iopm_gfn;
break;
case 32768 ... 65535: /* second 4KB page */
port -= 32768;
- io_bitmap = hvm_map_guest_frame_ro(iopm_gfn+1);
+ gfn = iopm_gfn + 1;
break;
default:
BUG();
break;
}
+ io_bitmap = hvm_map_guest_frame_ro(gfn);
if (io_bitmap == NULL) {
gdprintk(XENLOG_ERR,
"IOIO intercept: mapping of permission map failed\n");
@@ -882,6 +889,8 @@ nsvm_vmcb_guest_intercepts_ioio(paddr_t
enabled = test_bit(port, io_bitmap);
hvm_unmap_guest_frame(io_bitmap);
+ put_gfn(current->domain, gfn);
+
if (!enabled)
return NESTEDHVM_VMEXIT_HOST;
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/svm/svm.c
--- a/xen/arch/x86/hvm/svm/svm.c
+++ b/xen/arch/x86/hvm/svm/svm.c
@@ -244,9 +244,10 @@ static int svm_vmcb_restore(struct vcpu
{
if ( c->cr0 & X86_CR0_PG )
{
- mfn = mfn_x(gfn_to_mfn(v->domain, c->cr3 >> PAGE_SHIFT, &p2mt));
+ mfn = mfn_x(get_gfn(v->domain, c->cr3 >> PAGE_SHIFT, &p2mt));
if ( !p2m_is_ram(p2mt) || !get_page(mfn_to_page(mfn), v->domain) )
{
+ put_gfn(v->domain, c->cr3 >> PAGE_SHIFT);
gdprintk(XENLOG_ERR, "Invalid CR3 value=0x%"PRIx64"\n",
c->cr3);
return -EINVAL;
@@ -257,6 +258,8 @@ static int svm_vmcb_restore(struct vcpu
put_page(pagetable_get_page(v->arch.guest_table));
v->arch.guest_table = pagetable_from_pfn(mfn);
+ if ( c->cr0 & X86_CR0_PG )
+ put_gfn(v->domain, c->cr3 >> PAGE_SHIFT);
}
v->arch.hvm_vcpu.guest_cr[0] = c->cr0 | X86_CR0_ET;
@@ -1161,7 +1164,9 @@ static void svm_do_nested_pgfault(struct
p2m = p2m_get_p2m(v);
_d.gpa = gpa;
_d.qualification = 0;
- _d.mfn = mfn_x(gfn_to_mfn_type_p2m(p2m, gfn, &_d.p2mt, &p2ma,
p2m_query, NULL));
+ mfn = get_gfn_type_access(p2m, gfn, &_d.p2mt, &p2ma, p2m_query, NULL);
+ __put_gfn(p2m, gfn);
+ _d.mfn = mfn_x(mfn);
__trace_var(TRC_HVM_NPF, 0, sizeof(_d), &_d);
}
@@ -1181,7 +1186,8 @@ static void svm_do_nested_pgfault(struct
if ( p2m == NULL )
p2m = p2m_get_p2m(v);
/* Everything else is an error. */
- mfn = gfn_to_mfn_type_p2m(p2m, gfn, &p2mt, &p2ma, p2m_guest, NULL);
+ mfn = get_gfn_type_access(p2m, gfn, &p2mt, &p2ma, p2m_guest, NULL);
+ __put_gfn(p2m, gfn);
gdprintk(XENLOG_ERR,
"SVM violation gpa %#"PRIpaddr", mfn %#lx, type %i\n",
gpa, mfn_x(mfn), p2mt);
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/viridian.c
--- a/xen/arch/x86/hvm/viridian.c
+++ b/xen/arch/x86/hvm/viridian.c
@@ -134,12 +134,13 @@ void dump_apic_assist(struct vcpu *v)
static void enable_hypercall_page(struct domain *d)
{
unsigned long gmfn = d->arch.hvm_domain.viridian.hypercall_gpa.fields.pfn;
- unsigned long mfn = gmfn_to_mfn(d, gmfn);
+ unsigned long mfn = get_gfn_untyped(d, gmfn);
uint8_t *p;
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
{
+ put_gfn(d, gmfn);
gdprintk(XENLOG_WARNING, "Bad GMFN %lx (MFN %lx)\n", gmfn, mfn);
return;
}
@@ -162,13 +163,14 @@ static void enable_hypercall_page(struct
unmap_domain_page(p);
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, gmfn);
}
void initialize_apic_assist(struct vcpu *v)
{
struct domain *d = v->domain;
unsigned long gmfn = v->arch.hvm_vcpu.viridian.apic_assist.fields.pfn;
- unsigned long mfn = gmfn_to_mfn(d, gmfn);
+ unsigned long mfn = get_gfn_untyped(d, gmfn);
uint8_t *p;
/*
@@ -184,6 +186,7 @@ void initialize_apic_assist(struct vcpu
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
{
+ put_gfn(d, gmfn);
gdprintk(XENLOG_WARNING, "Bad GMFN %lx (MFN %lx)\n", gmfn, mfn);
return;
}
@@ -195,6 +198,7 @@ void initialize_apic_assist(struct vcpu
unmap_domain_page(p);
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, gmfn);
}
int wrmsr_viridian_regs(uint32_t idx, uint64_t val)
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -487,9 +487,10 @@ static int vmx_restore_cr0_cr3(
{
if ( cr0 & X86_CR0_PG )
{
- mfn = mfn_x(gfn_to_mfn(v->domain, cr3 >> PAGE_SHIFT, &p2mt));
+ mfn = mfn_x(get_gfn(v->domain, cr3 >> PAGE_SHIFT, &p2mt));
if ( !p2m_is_ram(p2mt) || !get_page(mfn_to_page(mfn), v->domain) )
{
+ put_gfn(v->domain, cr3 >> PAGE_SHIFT);
gdprintk(XENLOG_ERR, "Invalid CR3 value=0x%lx\n", cr3);
return -EINVAL;
}
@@ -499,6 +500,8 @@ static int vmx_restore_cr0_cr3(
put_page(pagetable_get_page(v->arch.guest_table));
v->arch.guest_table = pagetable_from_pfn(mfn);
+ if ( cr0 & X86_CR0_PG )
+ put_gfn(v->domain, cr3 >> PAGE_SHIFT);
}
v->arch.hvm_vcpu.guest_cr[0] = cr0 | X86_CR0_ET;
@@ -1007,9 +1010,12 @@ static void vmx_load_pdptrs(struct vcpu
if ( cr3 & 0x1fUL )
goto crash;
- mfn = mfn_x(gfn_to_mfn(v->domain, cr3 >> PAGE_SHIFT, &p2mt));
+ mfn = mfn_x(get_gfn(v->domain, cr3 >> PAGE_SHIFT, &p2mt));
if ( !p2m_is_ram(p2mt) )
+ {
+ put_gfn(v->domain, cr3 >> PAGE_SHIFT);
goto crash;
+ }
p = map_domain_page(mfn);
@@ -1037,6 +1043,7 @@ static void vmx_load_pdptrs(struct vcpu
vmx_vmcs_exit(v);
unmap_domain_page(p);
+ put_gfn(v->domain, cr3 >> PAGE_SHIFT);
return;
crash:
@@ -2090,7 +2097,7 @@ static void ept_handle_violation(unsigne
_d.gpa = gpa;
_d.qualification = qualification;
- _d.mfn = mfn_x(gfn_to_mfn_query(d, gfn, &_d.p2mt));
+ _d.mfn = mfn_x(get_gfn_query_unlocked(d, gfn, &_d.p2mt));
__trace_var(TRC_HVM_NPF, 0, sizeof(_d), &_d);
}
@@ -2106,7 +2113,7 @@ static void ept_handle_violation(unsigne
return;
/* Everything else is an error. */
- mfn = gfn_to_mfn_guest(d, gfn, &p2mt);
+ mfn = get_gfn_guest_unlocked(d, gfn, &p2mt);
gdprintk(XENLOG_ERR, "EPT violation %#lx (%c%c%c/%c%c%c), "
"gpa %#"PRIpaddr", mfn %#lx, type %i.\n",
qualification,
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c
+++ b/xen/arch/x86/hvm/vmx/vvmx.c
@@ -558,9 +558,12 @@ static void __map_io_bitmap(struct vcpu
index = vmcs_reg == IO_BITMAP_A ? 0 : 1;
if (nvmx->iobitmap[index])
- hvm_unmap_guest_frame (nvmx->iobitmap[index]);
+ hvm_unmap_guest_frame (nvmx->iobitmap[index]);
gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, vmcs_reg);
nvmx->iobitmap[index] = hvm_map_guest_frame_ro (gpa >> PAGE_SHIFT);
+ /* See comment in nestedsvm_vmcb_map re putting this gfn and
+ * liveness of the map it backs */
+ put_gfn(current->domain, gpa >> PAGE_SHIFT);
}
static inline void map_io_bitmap_all(struct vcpu *v)
@@ -577,12 +580,12 @@ static void nvmx_purge_vvmcs(struct vcpu
__clear_current_vvmcs(v);
if ( nvcpu->nv_vvmcxaddr != VMCX_EADDR )
- hvm_unmap_guest_frame (nvcpu->nv_vvmcx);
+ hvm_unmap_guest_frame(nvcpu->nv_vvmcx);
nvcpu->nv_vvmcx == NULL;
nvcpu->nv_vvmcxaddr = VMCX_EADDR;
for (i=0; i<2; i++) {
if ( nvmx->iobitmap[i] ) {
- hvm_unmap_guest_frame (nvmx->iobitmap[i]);
+ hvm_unmap_guest_frame(nvmx->iobitmap[i]);
nvmx->iobitmap[i] = NULL;
}
}
@@ -1138,6 +1141,9 @@ int nvmx_handle_vmptrld(struct cpu_user_
nvcpu->nv_vvmcx = hvm_map_guest_frame_rw (gpa >> PAGE_SHIFT);
nvcpu->nv_vvmcxaddr = gpa;
map_io_bitmap_all (v);
+ /* See comment in nestedsvm_vmcb_map regarding putting this
+ * gfn and liveness of the map that uses it */
+ put_gfn(current->domain, gpa >> PAGE_SHIFT);
}
vmreturn(regs, VMSUCCEED);
@@ -1199,6 +1205,7 @@ int nvmx_handle_vmclear(struct cpu_user_
if ( vvmcs )
__set_vvmcs(vvmcs, NVMX_LAUNCH_STATE, 0);
hvm_unmap_guest_frame(vvmcs);
+ put_gfn(current->domain, gpa >> PAGE_SHIFT);
}
vmreturn(regs, VMSUCCEED);
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm.c
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -663,13 +663,19 @@ int map_ldt_shadow_page(unsigned int off
return 0;
gmfn = l1e_get_pfn(l1e);
- mfn = gmfn_to_mfn(d, gmfn);
+ mfn = get_gfn_untyped(d, gmfn);
if ( unlikely(!mfn_valid(mfn)) )
+ {
+ put_gfn(d, gmfn);
return 0;
+ }
okay = get_page_and_type(mfn_to_page(mfn), d, PGT_seg_desc_page);
if ( unlikely(!okay) )
+ {
+ put_gfn(d, gmfn);
return 0;
+ }
nl1e = l1e_from_pfn(mfn, l1e_get_flags(l1e) | _PAGE_RW);
@@ -678,6 +684,7 @@ int map_ldt_shadow_page(unsigned int off
v->arch.pv_vcpu.shadow_ldt_mapcnt++;
spin_unlock(&v->arch.pv_vcpu.shadow_ldt_lock);
+ put_gfn(d, gmfn);
return 1;
}
@@ -1798,7 +1805,6 @@ static int mod_l1_entry(l1_pgentry_t *pl
{
l1_pgentry_t ol1e;
struct domain *pt_dom = pt_vcpu->domain;
- unsigned long mfn;
p2m_type_t p2mt;
int rc = 0;
@@ -1815,9 +1821,14 @@ static int mod_l1_entry(l1_pgentry_t *pl
if ( l1e_get_flags(nl1e) & _PAGE_PRESENT )
{
/* Translate foreign guest addresses. */
- mfn = mfn_x(gfn_to_mfn(pg_dom, l1e_get_pfn(nl1e), &p2mt));
+ unsigned long mfn, gfn;
+ gfn = l1e_get_pfn(nl1e);
+ mfn = mfn_x(get_gfn(pg_dom, gfn, &p2mt));
if ( !p2m_is_ram(p2mt) || unlikely(mfn == INVALID_MFN) )
+ {
+ put_gfn(pg_dom, gfn);
return -EINVAL;
+ }
ASSERT((mfn & ~(PADDR_MASK >> PAGE_SHIFT)) == 0);
nl1e = l1e_from_pfn(mfn, l1e_get_flags(nl1e));
@@ -1825,6 +1836,7 @@ static int mod_l1_entry(l1_pgentry_t *pl
{
MEM_LOG("Bad L1 flags %x",
l1e_get_flags(nl1e) & l1_disallow_mask(pt_dom));
+ put_gfn(pg_dom, gfn);
return -EINVAL;
}
@@ -1835,12 +1847,14 @@ static int mod_l1_entry(l1_pgentry_t *pl
if ( UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, pt_vcpu,
preserve_ad) )
return 0;
+ put_gfn(pg_dom, gfn);
return -EBUSY;
}
switch ( rc = get_page_from_l1e(nl1e, pt_dom, pg_dom) )
{
default:
+ put_gfn(pg_dom, gfn);
return rc;
case 0:
break;
@@ -1856,6 +1870,7 @@ static int mod_l1_entry(l1_pgentry_t *pl
ol1e = nl1e;
rc = -EBUSY;
}
+ put_gfn(pg_dom, gfn);
}
else if ( unlikely(!UPDATE_ENTRY(l1, pl1e, ol1e, nl1e, gl1mfn, pt_vcpu,
preserve_ad)) )
@@ -3023,7 +3038,7 @@ int do_mmuext_op(
if ( paging_mode_refcounts(pg_owner) )
break;
- mfn = gmfn_to_mfn(pg_owner, op.arg1.mfn);
+ mfn = get_gfn_untyped(pg_owner, op.arg1.mfn);
rc = get_page_and_type_from_pagenr(mfn, type, pg_owner, 0, 1);
okay = !rc;
if ( unlikely(!okay) )
@@ -3032,6 +3047,7 @@ int do_mmuext_op(
rc = -EAGAIN;
else if ( rc != -EAGAIN )
MEM_LOG("Error while pinning mfn %lx", mfn);
+ put_gfn(pg_owner, op.arg1.mfn);
break;
}
@@ -3040,6 +3056,7 @@ int do_mmuext_op(
if ( (rc = xsm_memory_pin_page(d, page)) != 0 )
{
put_page_and_type(page);
+ put_gfn(pg_owner, op.arg1.mfn);
okay = 0;
break;
}
@@ -3049,6 +3066,7 @@ int do_mmuext_op(
{
MEM_LOG("Mfn %lx already pinned", mfn);
put_page_and_type(page);
+ put_gfn(pg_owner, op.arg1.mfn);
okay = 0;
break;
}
@@ -3067,6 +3085,7 @@ int do_mmuext_op(
spin_unlock(&pg_owner->page_alloc_lock);
if ( drop_ref )
put_page_and_type(page);
+ put_gfn(pg_owner, op.arg1.mfn);
}
break;
@@ -3079,9 +3098,10 @@ int do_mmuext_op(
if ( paging_mode_refcounts(pg_owner) )
break;
- mfn = gmfn_to_mfn(pg_owner, op.arg1.mfn);
+ mfn = get_gfn_untyped(pg_owner, op.arg1.mfn);
if ( unlikely(!(okay = get_page_from_pagenr(mfn, pg_owner))) )
{
+ put_gfn(pg_owner, op.arg1.mfn);
MEM_LOG("Mfn %lx bad domain", mfn);
break;
}
@@ -3092,6 +3112,7 @@ int do_mmuext_op(
{
okay = 0;
put_page(page);
+ put_gfn(pg_owner, op.arg1.mfn);
MEM_LOG("Mfn %lx not pinned", mfn);
break;
}
@@ -3102,18 +3123,20 @@ int do_mmuext_op(
/* A page is dirtied when its pin status is cleared. */
paging_mark_dirty(pg_owner, mfn);
+ put_gfn(pg_owner, op.arg1.mfn);
break;
}
case MMUEXT_NEW_BASEPTR:
- okay = new_guest_cr3(gmfn_to_mfn(d, op.arg1.mfn));
+ okay = new_guest_cr3(get_gfn_untyped(d, op.arg1.mfn));
+ put_gfn(d, op.arg1.mfn);
break;
#ifdef __x86_64__
case MMUEXT_NEW_USER_BASEPTR: {
unsigned long old_mfn, mfn;
- mfn = gmfn_to_mfn(d, op.arg1.mfn);
+ mfn = get_gfn_untyped(d, op.arg1.mfn);
if ( mfn != 0 )
{
if ( paging_mode_refcounts(d) )
@@ -3123,6 +3146,7 @@ int do_mmuext_op(
mfn, PGT_root_page_table, d, 0, 0);
if ( unlikely(!okay) )
{
+ put_gfn(d, op.arg1.mfn);
MEM_LOG("Error while installing new mfn %lx", mfn);
break;
}
@@ -3130,6 +3154,7 @@ int do_mmuext_op(
old_mfn = pagetable_get_pfn(curr->arch.guest_table_user);
curr->arch.guest_table_user = pagetable_from_pfn(mfn);
+ put_gfn(d, op.arg1.mfn);
if ( old_mfn != 0 )
{
@@ -3247,11 +3272,12 @@ int do_mmuext_op(
unsigned long mfn;
unsigned char *ptr;
- mfn = gmfn_to_mfn(d, op.arg1.mfn);
+ mfn = get_gfn_untyped(d, op.arg1.mfn);
okay = !get_page_and_type_from_pagenr(
mfn, PGT_writable_page, d, 0, 0);
if ( unlikely(!okay) )
{
+ put_gfn(d, op.arg1.mfn);
MEM_LOG("Error while clearing mfn %lx", mfn);
break;
}
@@ -3264,6 +3290,7 @@ int do_mmuext_op(
fixunmap_domain_page(ptr);
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, op.arg1.mfn);
break;
}
@@ -3273,20 +3300,23 @@ int do_mmuext_op(
unsigned char *dst;
unsigned long src_mfn, mfn;
- src_mfn = gmfn_to_mfn(d, op.arg2.src_mfn);
+ src_mfn = get_gfn_untyped(d, op.arg2.src_mfn);
okay = get_page_from_pagenr(src_mfn, d);
if ( unlikely(!okay) )
{
+ put_gfn(d, op.arg2.src_mfn);
MEM_LOG("Error while copying from mfn %lx", src_mfn);
break;
}
- mfn = gmfn_to_mfn(d, op.arg1.mfn);
+ mfn = get_gfn_untyped(d, op.arg1.mfn);
okay = !get_page_and_type_from_pagenr(
mfn, PGT_writable_page, d, 0, 0);
if ( unlikely(!okay) )
{
+ put_gfn(d, op.arg1.mfn);
put_page(mfn_to_page(src_mfn));
+ put_gfn(d, op.arg2.src_mfn);
MEM_LOG("Error while copying to mfn %lx", mfn);
break;
}
@@ -3301,7 +3331,9 @@ int do_mmuext_op(
unmap_domain_page(src);
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, op.arg1.mfn);
put_page(mfn_to_page(src_mfn));
+ put_gfn(d, op.arg2.src_mfn);
break;
}
@@ -3489,14 +3521,14 @@ int do_mmu_update(
req.ptr -= cmd;
gmfn = req.ptr >> PAGE_SHIFT;
- mfn = mfn_x(gfn_to_mfn(pt_owner, gmfn, &p2mt));
+ mfn = mfn_x(get_gfn(pt_owner, gmfn, &p2mt));
if ( !p2m_is_valid(p2mt) )
- mfn = INVALID_MFN;
+ mfn = INVALID_MFN;
if ( p2m_is_paged(p2mt) )
{
p2m_mem_paging_populate(pg_owner, gmfn);
-
+ put_gfn(pt_owner, gmfn);
rc = -ENOENT;
break;
}
@@ -3504,6 +3536,7 @@ int do_mmu_update(
if ( unlikely(!get_page_from_pagenr(mfn, pt_owner)) )
{
MEM_LOG("Could not get page for normal update");
+ put_gfn(pt_owner, gmfn);
break;
}
@@ -3516,6 +3549,7 @@ int do_mmu_update(
if ( rc ) {
unmap_domain_page_with_cache(va, &mapcache);
put_page(page);
+ put_gfn(pt_owner, gmfn);
break;
}
@@ -3527,16 +3561,20 @@ int do_mmu_update(
{
l1_pgentry_t l1e = l1e_from_intpte(req.val);
p2m_type_t l1e_p2mt;
- gfn_to_mfn(pg_owner, l1e_get_pfn(l1e), &l1e_p2mt);
+ unsigned long l1egfn = l1e_get_pfn(l1e), l1emfn;
+
+ l1emfn = mfn_x(get_gfn(pg_owner, l1egfn, &l1e_p2mt));
if ( p2m_is_paged(l1e_p2mt) )
{
p2m_mem_paging_populate(pg_owner, l1e_get_pfn(l1e));
+ put_gfn(pg_owner, l1egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_paging_in_start == l1e_p2mt &&
!mfn_valid(mfn) )
{
+ put_gfn(pg_owner, l1egfn);
rc = -ENOENT;
break;
}
@@ -3553,7 +3591,10 @@ int do_mmu_update(
l1e_get_pfn(l1e),
0);
if ( rc )
+ {
+ put_gfn(pg_owner, l1egfn);
break;
+ }
}
}
#endif
@@ -3561,27 +3602,33 @@ int do_mmu_update(
rc = mod_l1_entry(va, l1e, mfn,
cmd == MMU_PT_UPDATE_PRESERVE_AD, v,
pg_owner);
+ put_gfn(pg_owner, l1egfn);
}
break;
case PGT_l2_page_table:
{
l2_pgentry_t l2e = l2e_from_intpte(req.val);
p2m_type_t l2e_p2mt;
- gfn_to_mfn(pg_owner, l2e_get_pfn(l2e), &l2e_p2mt);
+ unsigned long l2egfn = l2e_get_pfn(l2e), l2emfn;
+
+ l2emfn = mfn_x(get_gfn(pg_owner, l2egfn, &l2e_p2mt));
if ( p2m_is_paged(l2e_p2mt) )
{
- p2m_mem_paging_populate(pg_owner, l2e_get_pfn(l2e));
+ p2m_mem_paging_populate(pg_owner, l2egfn);
+ put_gfn(pg_owner, l2egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_paging_in_start == l2e_p2mt &&
!mfn_valid(mfn) )
{
+ put_gfn(pg_owner, l2egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_shared == l2e_p2mt )
{
+ put_gfn(pg_owner, l2egfn);
MEM_LOG("Unexpected attempt to map shared page.\n");
break;
}
@@ -3589,33 +3636,40 @@ int do_mmu_update(
rc = mod_l2_entry(va, l2e, mfn,
cmd == MMU_PT_UPDATE_PRESERVE_AD, v);
+ put_gfn(pg_owner, l2egfn);
}
break;
case PGT_l3_page_table:
{
l3_pgentry_t l3e = l3e_from_intpte(req.val);
p2m_type_t l3e_p2mt;
- gfn_to_mfn(pg_owner, l3e_get_pfn(l3e), &l3e_p2mt);
+ unsigned long l3egfn = l3e_get_pfn(l3e), l3emfn;
+
+ l3emfn = mfn_x(get_gfn(pg_owner, l3egfn, &l3e_p2mt));
if ( p2m_is_paged(l3e_p2mt) )
{
- p2m_mem_paging_populate(pg_owner, l3e_get_pfn(l3e));
+ p2m_mem_paging_populate(pg_owner, l3egfn);
+ put_gfn(pg_owner, l3egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_paging_in_start == l3e_p2mt &&
!mfn_valid(mfn) )
{
+ put_gfn(pg_owner, l3egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_shared == l3e_p2mt )
{
+ put_gfn(pg_owner, l3egfn);
MEM_LOG("Unexpected attempt to map shared page.\n");
break;
}
rc = mod_l3_entry(va, l3e, mfn,
cmd == MMU_PT_UPDATE_PRESERVE_AD, 1, v);
+ put_gfn(pg_owner, l3egfn);
}
break;
#if CONFIG_PAGING_LEVELS >= 4
@@ -3623,27 +3677,33 @@ int do_mmu_update(
{
l4_pgentry_t l4e = l4e_from_intpte(req.val);
p2m_type_t l4e_p2mt;
- gfn_to_mfn(pg_owner, l4e_get_pfn(l4e), &l4e_p2mt);
+ unsigned long l4egfn = l4e_get_pfn(l4e), l4emfn;
+
+ l4emfn = mfn_x(get_gfn(pg_owner, l4egfn, &l4e_p2mt));
if ( p2m_is_paged(l4e_p2mt) )
{
- p2m_mem_paging_populate(pg_owner, l4e_get_pfn(l4e));
+ p2m_mem_paging_populate(pg_owner, l4egfn);
+ put_gfn(pg_owner, l4egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_paging_in_start == l4e_p2mt &&
!mfn_valid(mfn) )
{
+ put_gfn(pg_owner, l4egfn);
rc = -ENOENT;
break;
}
else if ( p2m_ram_shared == l4e_p2mt )
{
+ put_gfn(pg_owner, l4egfn);
MEM_LOG("Unexpected attempt to map shared page.\n");
break;
}
rc = mod_l4_entry(va, l4e, mfn,
cmd == MMU_PT_UPDATE_PRESERVE_AD, 1, v);
+ put_gfn(pg_owner, l4egfn);
}
break;
#endif
@@ -3667,6 +3727,7 @@ int do_mmu_update(
unmap_domain_page_with_cache(va, &mapcache);
put_page(page);
+ put_gfn(pt_owner, gmfn);
}
break;
@@ -3753,10 +3814,11 @@ static int create_grant_pte_mapping(
adjust_guest_l1e(nl1e, d);
gmfn = pte_addr >> PAGE_SHIFT;
- mfn = gmfn_to_mfn(d, gmfn);
+ mfn = get_gfn_untyped(d, gmfn);
if ( unlikely(!get_page_from_pagenr(mfn, current->domain)) )
{
+ put_gfn(d, gmfn);
MEM_LOG("Could not get page for normal update");
return GNTST_general_error;
}
@@ -3794,6 +3856,7 @@ static int create_grant_pte_mapping(
failed:
unmap_domain_page(va);
put_page(page);
+ put_gfn(d, gmfn);
return rc;
}
@@ -3808,10 +3871,11 @@ static int destroy_grant_pte_mapping(
l1_pgentry_t ol1e;
gmfn = addr >> PAGE_SHIFT;
- mfn = gmfn_to_mfn(d, gmfn);
+ mfn = get_gfn_untyped(d, gmfn);
if ( unlikely(!get_page_from_pagenr(mfn, current->domain)) )
{
+ put_gfn(d, gmfn);
MEM_LOG("Could not get page for normal update");
return GNTST_general_error;
}
@@ -3863,6 +3927,7 @@ static int destroy_grant_pte_mapping(
failed:
unmap_domain_page(va);
put_page(page);
+ put_gfn(d, gmfn);
return rc;
}
@@ -4054,9 +4119,10 @@ static int replace_grant_p2m_mapping(
if ( new_addr != 0 || (flags & GNTMAP_contains_pte) )
return GNTST_general_error;
- old_mfn = gfn_to_mfn(d, gfn, &type);
+ old_mfn = get_gfn(d, gfn, &type);
if ( !p2m_is_grant(type) || mfn_x(old_mfn) != frame )
{
+ put_gfn(d, gfn);
gdprintk(XENLOG_WARNING,
"replace_grant_p2m_mapping: old mapping invalid (type %d, mfn
%lx, frame %lx)\n",
type, mfn_x(old_mfn), frame);
@@ -4064,6 +4130,7 @@ static int replace_grant_p2m_mapping(
}
guest_physmap_remove_page(d, gfn, frame, PAGE_ORDER_4K);
+ put_gfn(d, gfn);
return GNTST_okay;
}
@@ -4444,15 +4511,20 @@ long set_gdt(struct vcpu *v,
struct domain *d = v->domain;
/* NB. There are 512 8-byte entries per GDT page. */
int i, nr_pages = (entries + 511) / 512;
- unsigned long mfn;
+ unsigned long mfn, *pfns;
if ( entries > FIRST_RESERVED_GDT_ENTRY )
return -EINVAL;
+ pfns = xmalloc_array(unsigned long, nr_pages);
+ if ( !pfns )
+ return -ENOMEM;
+
/* Check the pages in the new GDT. */
for ( i = 0; i < nr_pages; i++ )
{
- mfn = frames[i] = gmfn_to_mfn(d, frames[i]);
+ pfns[i] = frames[i];
+ mfn = frames[i] = get_gfn_untyped(d, frames[i]);
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_seg_desc_page) )
goto fail;
@@ -4468,13 +4540,19 @@ long set_gdt(struct vcpu *v,
v->arch.pv_vcpu.gdt_frames[i] = frames[i];
l1e_write(&v->arch.perdomain_ptes[i],
l1e_from_pfn(frames[i], __PAGE_HYPERVISOR));
+ put_gfn(d, pfns[i]);
}
+ xfree(pfns);
return 0;
fail:
while ( i-- > 0 )
+ {
put_page_and_type(mfn_to_page(frames[i]));
+ put_gfn(d, pfns[i]);
+ }
+ xfree(pfns);
return -EINVAL;
}
@@ -4518,15 +4596,21 @@ long do_update_descriptor(u64 pa, u64 de
*(u64 *)&d = desc;
- mfn = gmfn_to_mfn(dom, gmfn);
+ mfn = get_gfn_untyped(dom, gmfn);
if ( (((unsigned int)pa % sizeof(struct desc_struct)) != 0) ||
!mfn_valid(mfn) ||
!check_descriptor(dom, &d) )
+ {
+ put_gfn(dom, gmfn);
return -EINVAL;
+ }
page = mfn_to_page(mfn);
if ( unlikely(!get_page(page, dom)) )
+ {
+ put_gfn(dom, gmfn);
return -EINVAL;
+ }
/* Check if the given frame is in use in an unsafe context. */
switch ( page->u.inuse.type_info & PGT_type_mask )
@@ -4554,6 +4638,7 @@ long do_update_descriptor(u64 pa, u64 de
out:
put_page(page);
+ put_gfn(dom, gmfn);
return ret;
}
@@ -4595,6 +4680,7 @@ static int handle_iomem_range(unsigned l
long arch_memory_op(int op, XEN_GUEST_HANDLE(void) arg)
{
struct page_info *page = NULL;
+ unsigned long gfn = 0; /* gcc ... */
int rc;
switch ( op )
@@ -4652,11 +4738,13 @@ long arch_memory_op(int op, XEN_GUEST_HA
case XENMAPSPACE_gmfn:
{
p2m_type_t p2mt;
-
- xatp.idx = mfn_x(gfn_to_mfn_unshare(d, xatp.idx, &p2mt));
+ gfn = xatp.idx;
+
+ xatp.idx = mfn_x(get_gfn_unshare(d, xatp.idx, &p2mt));
/* If the page is still shared, exit early */
if ( p2m_is_shared(p2mt) )
{
+ put_gfn(d, gfn);
rcu_unlock_domain(d);
return -ENOMEM;
}
@@ -4674,6 +4762,8 @@ long arch_memory_op(int op, XEN_GUEST_HA
{
if ( page )
put_page(page);
+ if ( xatp.space == XENMAPSPACE_gmfn )
+ put_gfn(d, gfn);
rcu_unlock_domain(d);
return -EINVAL;
}
@@ -4684,7 +4774,7 @@ long arch_memory_op(int op, XEN_GUEST_HA
put_page(page);
/* Remove previously mapped page if it was present. */
- prev_mfn = gmfn_to_mfn(d, xatp.gpfn);
+ prev_mfn = get_gfn_untyped(d, xatp.gpfn);
if ( mfn_valid(prev_mfn) )
{
if ( is_xen_heap_mfn(prev_mfn) )
@@ -4694,6 +4784,8 @@ long arch_memory_op(int op, XEN_GUEST_HA
/* Normal domain memory is freed, to avoid leaking memory. */
guest_remove_page(d, xatp.gpfn);
}
+ /* In the XENMAPSPACE_gmfn case we still hold a ref on the old page. */
+ put_gfn(d, xatp.gpfn);
/* Unmap from old location, if any. */
gpfn = get_gpfn_from_mfn(mfn);
@@ -4704,6 +4796,9 @@ long arch_memory_op(int op, XEN_GUEST_HA
/* Map at new location. */
rc = guest_physmap_add_page(d, xatp.gpfn, mfn, PAGE_ORDER_4K);
+ /* In the XENMAPSPACE_gmfn, we took a ref and locked the p2m at the
top */
+ if ( xatp.space == XENMAPSPACE_gmfn )
+ put_gfn(d, gfn);
domain_unlock(d);
rcu_unlock_domain(d);
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/guest_walk.c
--- a/xen/arch/x86/mm/guest_walk.c
+++ b/xen/arch/x86/mm/guest_walk.c
@@ -86,30 +86,35 @@ static uint32_t set_ad_bits(void *guest_
return 0;
}
+/* If the map is non-NULL, we leave this function having
+ * called get_gfn, you need to call put_gfn. */
static inline void *map_domain_gfn(struct p2m_domain *p2m,
gfn_t gfn,
mfn_t *mfn,
p2m_type_t *p2mt,
uint32_t *rc)
{
- p2m_access_t a;
+ p2m_access_t p2ma;
/* Translate the gfn, unsharing if shared */
- *mfn = gfn_to_mfn_type_p2m(p2m, gfn_x(gfn), p2mt, &a, p2m_unshare, NULL);
+ *mfn = get_gfn_type_access(p2m, gfn_x(gfn), p2mt, &p2ma, p2m_unshare,
NULL);
if ( p2m_is_paging(*p2mt) )
{
ASSERT(!p2m_is_nestedp2m(p2m));
p2m_mem_paging_populate(p2m->domain, gfn_x(gfn));
+ __put_gfn(p2m, gfn_x(gfn));
*rc = _PAGE_PAGED;
return NULL;
}
if ( p2m_is_shared(*p2mt) )
{
+ __put_gfn(p2m, gfn_x(gfn));
*rc = _PAGE_SHARED;
return NULL;
}
if ( !p2m_is_ram(*p2mt) )
{
+ __put_gfn(p2m, gfn_x(gfn));
*rc |= _PAGE_PRESENT;
return NULL;
}
@@ -120,6 +125,9 @@ static inline void *map_domain_gfn(struc
/* Walk the guest pagetables, after the manner of a hardware walker. */
+/* Because the walk is essentially random, it can cause a deadlock
+ * warning in the p2m locking code. Highly unlikely this is an actual
+ * deadlock, because who would walk page table in the opposite order? */
uint32_t
guest_walk_tables(struct vcpu *v, struct p2m_domain *p2m,
unsigned long va, walk_t *gw,
@@ -288,7 +296,6 @@ guest_walk_tables(struct vcpu *v, struct
#endif
rc |= _PAGE_INVALID_BITS;
}
-
/* Increment the pfn by the right number of 4k pages.
* Mask out PAT and invalid bits. */
start = _gfn((gfn_x(start) & ~GUEST_L2_GFN_MASK) +
@@ -347,12 +354,24 @@ set_ad:
out:
#if GUEST_PAGING_LEVELS == 4
- if ( l3p ) unmap_domain_page(l3p);
+ if ( l3p )
+ {
+ unmap_domain_page(l3p);
+ __put_gfn(p2m, gfn_x(guest_l4e_get_gfn(gw->l4e)));
+ }
#endif
#if GUEST_PAGING_LEVELS >= 3
- if ( l2p ) unmap_domain_page(l2p);
+ if ( l2p )
+ {
+ unmap_domain_page(l2p);
+ __put_gfn(p2m, gfn_x(guest_l3e_get_gfn(gw->l3e)));
+ }
#endif
- if ( l1p ) unmap_domain_page(l1p);
+ if ( l1p )
+ {
+ unmap_domain_page(l1p);
+ __put_gfn(p2m, gfn_x(guest_l2e_get_gfn(gw->l2e)));
+ }
return rc;
}
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/hap/guest_walk.c
--- a/xen/arch/x86/mm/hap/guest_walk.c
+++ b/xen/arch/x86/mm/hap/guest_walk.c
@@ -56,26 +56,30 @@ unsigned long hap_p2m_ga_to_gfn(GUEST_PA
p2m_type_t p2mt;
p2m_access_t p2ma;
walk_t gw;
+ unsigned long top_gfn;
/* Get the top-level table's MFN */
- top_mfn = gfn_to_mfn_type_p2m(p2m, cr3 >> PAGE_SHIFT,
- &p2mt, &p2ma, p2m_unshare, NULL);
+ top_gfn = cr3 >> PAGE_SHIFT;
+ top_mfn = get_gfn_type_access(p2m, top_gfn, &p2mt, &p2ma, p2m_unshare,
NULL);
if ( p2m_is_paging(p2mt) )
{
ASSERT(!p2m_is_nestedp2m(p2m));
p2m_mem_paging_populate(p2m->domain, cr3 >> PAGE_SHIFT);
pfec[0] = PFEC_page_paged;
+ __put_gfn(p2m, top_gfn);
return INVALID_GFN;
}
if ( p2m_is_shared(p2mt) )
{
pfec[0] = PFEC_page_shared;
+ __put_gfn(p2m, top_gfn);
return INVALID_GFN;
}
if ( !p2m_is_ram(p2mt) )
{
pfec[0] &= ~PFEC_page_present;
+ __put_gfn(p2m, top_gfn);
return INVALID_GFN;
}
@@ -87,26 +91,31 @@ unsigned long hap_p2m_ga_to_gfn(GUEST_PA
#endif
missing = guest_walk_tables(v, p2m, ga, &gw, pfec[0], top_mfn, top_map);
unmap_domain_page(top_map);
+ __put_gfn(p2m, top_gfn);
/* Interpret the answer */
if ( missing == 0 )
{
gfn_t gfn = guest_l1e_get_gfn(gw.l1e);
- gfn_to_mfn_type_p2m(p2m, gfn_x(gfn), &p2mt, &p2ma, p2m_unshare, NULL);
+ (void)get_gfn_type_access(p2m, gfn_x(gfn), &p2mt, &p2ma, p2m_unshare,
NULL);
if ( p2m_is_paging(p2mt) )
{
ASSERT(!p2m_is_nestedp2m(p2m));
p2m_mem_paging_populate(p2m->domain, gfn_x(gfn));
pfec[0] = PFEC_page_paged;
+ __put_gfn(p2m, gfn_x(gfn));
return INVALID_GFN;
}
if ( p2m_is_shared(p2mt) )
{
pfec[0] = PFEC_page_shared;
+ __put_gfn(p2m, gfn_x(gfn));
return INVALID_GFN;
}
+ __put_gfn(p2m, gfn_x(gfn));
+
if ( page_order )
*page_order = guest_walk_to_page_order(&gw);
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/hap/nested_hap.c
--- a/xen/arch/x86/mm/hap/nested_hap.c
+++ b/xen/arch/x86/mm/hap/nested_hap.c
@@ -146,22 +146,29 @@ nestedhap_walk_L0_p2m(struct p2m_domain
mfn_t mfn;
p2m_type_t p2mt;
p2m_access_t p2ma;
+ int rc;
/* walk L0 P2M table */
- mfn = gfn_to_mfn_type_p2m(p2m, L1_gpa >> PAGE_SHIFT, &p2mt, &p2ma,
+ mfn = get_gfn_type_access(p2m, L1_gpa >> PAGE_SHIFT, &p2mt, &p2ma,
p2m_query, page_order);
+ rc = NESTEDHVM_PAGEFAULT_MMIO;
if ( p2m_is_mmio(p2mt) )
- return NESTEDHVM_PAGEFAULT_MMIO;
+ goto out;
+ rc = NESTEDHVM_PAGEFAULT_ERROR;
if ( p2m_is_paging(p2mt) || p2m_is_shared(p2mt) || !p2m_is_ram(p2mt) )
- return NESTEDHVM_PAGEFAULT_ERROR;
+ goto out;
+ rc = NESTEDHVM_PAGEFAULT_ERROR;
if ( !mfn_valid(mfn) )
- return NESTEDHVM_PAGEFAULT_ERROR;
+ goto out;
*L0_gpa = (mfn_x(mfn) << PAGE_SHIFT) + (L1_gpa & ~PAGE_MASK);
- return NESTEDHVM_PAGEFAULT_DONE;
+ rc = NESTEDHVM_PAGEFAULT_DONE;
+out:
+ __put_gfn(p2m, L1_gpa >> PAGE_SHIFT);
+ return rc;
}
/* This function uses L2_gpa to walk the P2M page table in L1. If the
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/mem_event.c
--- a/xen/arch/x86/mm/mem_event.c
+++ b/xen/arch/x86/mm/mem_event.c
@@ -47,7 +47,7 @@ static int mem_event_enable(struct domai
unsigned long ring_addr = mec->ring_addr;
unsigned long shared_addr = mec->shared_addr;
l1_pgentry_t l1e;
- unsigned long gfn;
+ unsigned long shared_gfn = 0, ring_gfn = 0; /* gcc ... */
p2m_type_t p2mt;
mfn_t ring_mfn;
mfn_t shared_mfn;
@@ -60,23 +60,36 @@ static int mem_event_enable(struct domai
/* Get MFN of ring page */
guest_get_eff_l1e(v, ring_addr, &l1e);
- gfn = l1e_get_pfn(l1e);
- ring_mfn = gfn_to_mfn(dom_mem_event, gfn, &p2mt);
+ ring_gfn = l1e_get_pfn(l1e);
+ /* We're grabbing these two in an order that could deadlock
+ * dom0 if 1. it were an hvm 2. there were two concurrent
+ * enables 3. the two gfn's in each enable criss-crossed
+ * 2MB regions. Duly noted.... */
+ ring_mfn = get_gfn(dom_mem_event, ring_gfn, &p2mt);
if ( unlikely(!mfn_valid(mfn_x(ring_mfn))) )
+ {
+ put_gfn(dom_mem_event, ring_gfn);
return -EINVAL;
+ }
/* Get MFN of shared page */
guest_get_eff_l1e(v, shared_addr, &l1e);
- gfn = l1e_get_pfn(l1e);
- shared_mfn = gfn_to_mfn(dom_mem_event, gfn, &p2mt);
+ shared_gfn = l1e_get_pfn(l1e);
+ shared_mfn = get_gfn(dom_mem_event, shared_gfn, &p2mt);
if ( unlikely(!mfn_valid(mfn_x(shared_mfn))) )
+ {
+ put_gfn(dom_mem_event, ring_gfn);
+ put_gfn(dom_mem_event, shared_gfn);
return -EINVAL;
+ }
/* Map ring and shared pages */
med->ring_page = map_domain_page(mfn_x(ring_mfn));
med->shared_page = map_domain_page(mfn_x(shared_mfn));
+ put_gfn(dom_mem_event, ring_gfn);
+ put_gfn(dom_mem_event, shared_gfn);
/* Allocate event channel */
rc = alloc_unbound_xen_event_channel(d->vcpu[0],
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/mem_sharing.c
--- a/xen/arch/x86/mm/mem_sharing.c
+++ b/xen/arch/x86/mm/mem_sharing.c
@@ -227,7 +227,7 @@ static void mem_sharing_audit(void)
g->domain, g->gfn, mfn_x(e->mfn));
continue;
}
- mfn = gfn_to_mfn(d, g->gfn, &t);
+ mfn = get_gfn_unlocked(d, g->gfn, &t);
if(mfn_x(mfn) != mfn_x(e->mfn))
MEM_SHARING_DEBUG("Incorrect P2M for d=%d, PFN=%lx."
"Expecting MFN=%ld, got %ld\n",
@@ -335,7 +335,7 @@ int mem_sharing_debug_gfn(struct domain
p2m_type_t p2mt;
mfn_t mfn;
- mfn = gfn_to_mfn(d, gfn, &p2mt);
+ mfn = get_gfn_unlocked(d, gfn, &p2mt);
printk("Debug for domain=%d, gfn=%lx, ",
d->domain_id,
@@ -460,7 +460,7 @@ int mem_sharing_nominate_page(struct dom
*phandle = 0UL;
shr_lock();
- mfn = gfn_to_mfn(d, gfn, &p2mt);
+ mfn = get_gfn(d, gfn, &p2mt);
/* Check if mfn is valid */
ret = -EINVAL;
@@ -524,6 +524,7 @@ int mem_sharing_nominate_page(struct dom
ret = 0;
out:
+ put_gfn(d, gfn);
shr_unlock();
return ret;
}
@@ -593,14 +594,18 @@ int mem_sharing_unshare_page(struct doma
shr_handle_t handle;
struct list_head *le;
+ /* Remove the gfn_info from the list */
+
+ /* This is one of the reasons why we can't enforce ordering
+ * between shr_lock and p2m fine-grained locks in mm-lock.
+ * Callers may walk in here already holding the lock for this gfn */
shr_lock();
mem_sharing_audit();
-
- /* Remove the gfn_info from the list */
- mfn = gfn_to_mfn(d, gfn, &p2mt);
+ mfn = get_gfn(d, gfn, &p2mt);
/* Has someone already unshared it? */
if (!p2m_is_shared(p2mt)) {
+ put_gfn(d, gfn);
shr_unlock();
return 0;
}
@@ -634,6 +639,7 @@ gfn_found:
/* Even though we don't allocate a private page, we have to account
* for the MFN that originally backed this PFN. */
atomic_dec(&nr_saved_mfns);
+ put_gfn(d, gfn);
shr_unlock();
put_page_and_type(page);
if(last_gfn &&
@@ -653,6 +659,7 @@ gfn_found:
/* We've failed to obtain memory for private page. Need to re-add the
* gfn_info to relevant list */
list_add(&gfn_info->list, &hash_entry->gfns);
+ put_gfn(d, gfn);
shr_unlock();
return -ENOMEM;
}
@@ -663,6 +670,13 @@ gfn_found:
unmap_domain_page(s);
unmap_domain_page(t);
+ /* NOTE: set_shared_p2m_entry will switch the underlying mfn. If
+ * we do get_page withing get_gfn, the correct sequence here
+ * should be
+ get_page(page);
+ put_page(old_page);
+ * so that the ref to the old page is dropped, and a ref to
+ * the new page is obtained to later be dropped in put_gfn */
BUG_ON(set_shared_p2m_entry(d, gfn, page_to_mfn(page)) == 0);
put_page_and_type(old_page);
@@ -683,6 +697,7 @@ private_page_found:
/* Update m2p entry */
set_gpfn_from_mfn(mfn_x(page_to_mfn(page)), gfn);
+ put_gfn(d, gfn);
shr_unlock();
return 0;
}
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/p2m-pod.c
--- a/xen/arch/x86/mm/p2m-pod.c
+++ b/xen/arch/x86/mm/p2m-pod.c
@@ -531,9 +531,10 @@ p2m_pod_decrease_reservation(struct doma
/* FIXME: Add contiguous; query for PSE entries? */
for ( i=0; i<(1<<order); i++)
{
+ p2m_access_t a;
p2m_type_t t;
- gfn_to_mfn_query(d, gpfn + i, &t);
+ (void)p2m->get_entry(p2m, gpfn + i, &t, &a, p2m_query, NULL);
if ( t == p2m_populate_on_demand )
pod++;
@@ -572,8 +573,9 @@ p2m_pod_decrease_reservation(struct doma
{
mfn_t mfn;
p2m_type_t t;
+ p2m_access_t a;
- mfn = gfn_to_mfn_query(d, gpfn + i, &t);
+ mfn = p2m->get_entry(p2m, gpfn + i, &t, &a, p2m_query, NULL);
if ( t == p2m_populate_on_demand )
{
set_p2m_entry(p2m, gpfn + i, _mfn(INVALID_MFN), 0, p2m_invalid,
p2m->default_access);
@@ -653,8 +655,8 @@ p2m_pod_zero_check_superpage(struct p2m_
* and aligned, and mapping them. */
for ( i=0; i<SUPERPAGE_PAGES; i++ )
{
-
- mfn = gfn_to_mfn_query(d, gfn + i, &type);
+ p2m_access_t a;
+ mfn = p2m->get_entry(p2m, gfn + i, &type, &a, p2m_query, NULL);
if ( i == 0 )
{
@@ -782,7 +784,8 @@ p2m_pod_zero_check(struct p2m_domain *p2
/* First, get the gfn list, translate to mfns, and map the pages. */
for ( i=0; i<count; i++ )
{
- mfns[i] = gfn_to_mfn_query(d, gfns[i], types + i);
+ p2m_access_t a;
+ mfns[i] = p2m->get_entry(p2m, gfns[i], types + i, &a, p2m_query, NULL);
/* If this is ram, and not a pagetable or from the xen heap, and
probably not mapped
elsewhere, map it; otherwise, skip. */
if ( p2m_is_ram(types[i])
@@ -923,7 +926,8 @@ p2m_pod_emergency_sweep(struct p2m_domai
/* FIXME: Figure out how to avoid superpages */
for ( i=p2m->pod.reclaim_single; i > 0 ; i-- )
{
- gfn_to_mfn_query(p2m->domain, i, &t );
+ p2m_access_t a;
+ (void)p2m->get_entry(p2m, i, &t, &a, p2m_query, NULL);
if ( p2m_is_ram(t) )
{
gfns[j] = i;
@@ -1112,7 +1116,8 @@ guest_physmap_mark_populate_on_demand(st
/* Make sure all gpfns are unused */
for ( i = 0; i < (1UL << order); i++ )
{
- omfn = gfn_to_mfn_query(d, gfn + i, &ot);
+ p2m_access_t a;
+ omfn = p2m->get_entry(p2m, gfn + i, &ot, &a, p2m_query, NULL);
if ( p2m_is_ram(ot) )
{
printk("%s: gfn_to_mfn returned type %d!\n",
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/p2m.c
--- a/xen/arch/x86/mm/p2m.c
+++ b/xen/arch/x86/mm/p2m.c
@@ -144,7 +144,7 @@ void p2m_change_entry_type_global(struct
p2m_unlock(p2m);
}
-mfn_t gfn_to_mfn_type_p2m(struct p2m_domain *p2m, unsigned long gfn,
+mfn_t get_gfn_type_access(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order)
{
@@ -343,28 +343,27 @@ void p2m_teardown(struct p2m_domain *p2m
#ifdef __x86_64__
unsigned long gfn;
p2m_type_t t;
- p2m_access_t a;
mfn_t mfn;
#endif
if (p2m == NULL)
return;
+ p2m_lock(p2m);
+
#ifdef __x86_64__
for ( gfn=0; gfn < p2m->max_mapped_pfn; gfn++ )
{
- mfn = gfn_to_mfn_type_p2m(p2m, gfn, &t, &a, p2m_query, NULL);
+ p2m_access_t a;
+ mfn = p2m->get_entry(p2m, gfn, &t, &a, p2m_query, NULL);
if ( mfn_valid(mfn) && (t == p2m_ram_shared) )
{
ASSERT(!p2m_is_nestedp2m(p2m));
BUG_ON(mem_sharing_unshare_page(d, gfn, MEM_SHARING_DESTROY_GFN));
}
-
}
#endif
- p2m_lock(p2m);
-
p2m->phys_table = pagetable_null();
while ( (pg = page_list_remove_head(&p2m->pages)) )
@@ -454,6 +453,7 @@ guest_physmap_add_entry(struct domain *d
struct p2m_domain *p2m = p2m_get_hostp2m(d);
unsigned long i, ogfn;
p2m_type_t ot;
+ p2m_access_t a;
mfn_t omfn;
int pod_count = 0;
int rc = 0;
@@ -489,12 +489,13 @@ guest_physmap_add_entry(struct domain *d
/* First, remove m->p mappings for existing p->m mappings */
for ( i = 0; i < (1UL << page_order); i++ )
{
- omfn = gfn_to_mfn_query(d, gfn + i, &ot);
+ omfn = p2m->get_entry(p2m, gfn + i, &ot, &a, p2m_query, NULL);
if ( p2m_is_grant(ot) )
{
/* Really shouldn't be unmapping grant maps this way */
domain_crash(d);
p2m_unlock(p2m);
+
return -EINVAL;
}
else if ( p2m_is_ram(ot) )
@@ -528,7 +529,7 @@ guest_physmap_add_entry(struct domain *d
* address */
P2M_DEBUG("aliased! mfn=%#lx, old gfn=%#lx, new gfn=%#lx\n",
mfn + i, ogfn, gfn + i);
- omfn = gfn_to_mfn_query(d, ogfn, &ot);
+ omfn = p2m->get_entry(p2m, ogfn, &ot, &a, p2m_query, NULL);
if ( p2m_is_ram(ot) )
{
ASSERT(mfn_valid(omfn));
@@ -577,6 +578,7 @@ guest_physmap_add_entry(struct domain *d
p2m_type_t p2m_change_type(struct domain *d, unsigned long gfn,
p2m_type_t ot, p2m_type_t nt)
{
+ p2m_access_t a;
p2m_type_t pt;
mfn_t mfn;
struct p2m_domain *p2m = p2m_get_hostp2m(d);
@@ -585,7 +587,7 @@ p2m_type_t p2m_change_type(struct domain
p2m_lock(p2m);
- mfn = gfn_to_mfn_query(d, gfn, &pt);
+ mfn = p2m->get_entry(p2m, gfn, &pt, &a, p2m_query, NULL);
if ( pt == ot )
set_p2m_entry(p2m, gfn, mfn, PAGE_ORDER_4K, nt, p2m->default_access);
@@ -600,6 +602,7 @@ void p2m_change_type_range(struct domain
unsigned long start, unsigned long end,
p2m_type_t ot, p2m_type_t nt)
{
+ p2m_access_t a;
p2m_type_t pt;
unsigned long gfn;
mfn_t mfn;
@@ -612,7 +615,7 @@ void p2m_change_type_range(struct domain
for ( gfn = start; gfn < end; gfn++ )
{
- mfn = gfn_to_mfn_query(d, gfn, &pt);
+ mfn = p2m->get_entry(p2m, gfn, &pt, &a, p2m_query, NULL);
if ( pt == ot )
set_p2m_entry(p2m, gfn, mfn, PAGE_ORDER_4K, nt,
p2m->default_access);
}
@@ -629,6 +632,7 @@ int
set_mmio_p2m_entry(struct domain *d, unsigned long gfn, mfn_t mfn)
{
int rc = 0;
+ p2m_access_t a;
p2m_type_t ot;
mfn_t omfn;
struct p2m_domain *p2m = p2m_get_hostp2m(d);
@@ -637,7 +641,7 @@ set_mmio_p2m_entry(struct domain *d, uns
return 0;
p2m_lock(p2m);
- omfn = gfn_to_mfn_query(d, gfn, &ot);
+ omfn = p2m->get_entry(p2m, gfn, &ot, &a, p2m_query, NULL);
if ( p2m_is_grant(ot) )
{
p2m_unlock(p2m);
@@ -657,7 +661,7 @@ set_mmio_p2m_entry(struct domain *d, uns
if ( 0 == rc )
gdprintk(XENLOG_ERR,
"set_mmio_p2m_entry: set_p2m_entry failed! mfn=%08lx\n",
- mfn_x(gfn_to_mfn_query(d, gfn, &ot)));
+ mfn_x(get_gfn_query_unlocked(p2m->domain, gfn, &ot)));
return rc;
}
@@ -666,6 +670,7 @@ clear_mmio_p2m_entry(struct domain *d, u
{
int rc = 0;
mfn_t mfn;
+ p2m_access_t a;
p2m_type_t t;
struct p2m_domain *p2m = p2m_get_hostp2m(d);
@@ -673,7 +678,7 @@ clear_mmio_p2m_entry(struct domain *d, u
return 0;
p2m_lock(p2m);
- mfn = gfn_to_mfn_query(d, gfn, &t);
+ mfn = p2m->get_entry(p2m, gfn, &t, &a, p2m_query, NULL);
/* Do not use mfn_valid() here as it will usually fail for MMIO pages. */
if ( (INVALID_MFN == mfn_x(mfn)) || (t != p2m_mmio_direct) )
@@ -696,6 +701,7 @@ set_shared_p2m_entry(struct domain *d, u
{
struct p2m_domain *p2m = p2m_get_hostp2m(d);
int rc = 0;
+ p2m_access_t a;
p2m_type_t ot;
mfn_t omfn;
@@ -703,7 +709,7 @@ set_shared_p2m_entry(struct domain *d, u
return 0;
p2m_lock(p2m);
- omfn = gfn_to_mfn_query(p2m->domain, gfn, &ot);
+ omfn = p2m->get_entry(p2m, gfn, &ot, &a, p2m_query, NULL);
/* At the moment we only allow p2m change if gfn has already been made
* sharable first */
ASSERT(p2m_is_shared(ot));
@@ -717,7 +723,7 @@ set_shared_p2m_entry(struct domain *d, u
if ( 0 == rc )
gdprintk(XENLOG_ERR,
"set_shared_p2m_entry: set_p2m_entry failed! mfn=%08lx\n",
- mfn_x(gfn_to_mfn_query(d, gfn, &ot)));
+ mfn_x(get_gfn_query_unlocked(p2m->domain, gfn, &ot)));
return rc;
}
@@ -1168,7 +1174,7 @@ int p2m_set_mem_access(struct domain *d,
{
struct p2m_domain *p2m = p2m_get_hostp2m(d);
unsigned long pfn;
- p2m_access_t a;
+ p2m_access_t a, _a;
p2m_type_t t;
mfn_t mfn;
int rc = 0;
@@ -1202,7 +1208,7 @@ int p2m_set_mem_access(struct domain *d,
p2m_lock(p2m);
for ( pfn = start_pfn; pfn < start_pfn + nr; pfn++ )
{
- mfn = gfn_to_mfn_query(d, pfn, &t);
+ mfn = p2m->get_entry(p2m, pfn, &t, &_a, p2m_query, NULL);
if ( p2m->set_entry(p2m, pfn, mfn, PAGE_ORDER_4K, t, a) == 0 )
{
rc = -ENOMEM;
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/shadow/common.c
--- a/xen/arch/x86/mm/shadow/common.c
+++ b/xen/arch/x86/mm/shadow/common.c
@@ -3676,7 +3676,7 @@ int shadow_track_dirty_vram(struct domai
/* Iterate over VRAM to track dirty bits. */
for ( i = 0; i < nr; i++ ) {
- mfn_t mfn = gfn_to_mfn_query(d, begin_pfn + i, &t);
+ mfn_t mfn = get_gfn_query(d, begin_pfn + i, &t);
struct page_info *page;
int dirty = 0;
paddr_t sl1ma = dirty_vram->sl1ma[i];
@@ -3741,6 +3741,8 @@ int shadow_track_dirty_vram(struct domai
}
}
+ put_gfn(d, begin_pfn + i);
+
if ( dirty )
{
dirty_vram->dirty_bitmap[i / 8] |= 1 << (i % 8);
@@ -3761,7 +3763,7 @@ int shadow_track_dirty_vram(struct domai
/* was clean for more than two seconds, try to disable guest
* write access */
for ( i = begin_pfn; i < end_pfn; i++ ) {
- mfn_t mfn = gfn_to_mfn_query(d, i, &t);
+ mfn_t mfn = get_gfn_query_unlocked(d, i, &t);
if (mfn_x(mfn) != INVALID_MFN)
flush_tlb |= sh_remove_write_access(d->vcpu[0], mfn,
1, 0);
}
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/shadow/multi.c
--- a/xen/arch/x86/mm/shadow/multi.c
+++ b/xen/arch/x86/mm/shadow/multi.c
@@ -2265,7 +2265,7 @@ static int validate_gl4e(struct vcpu *v,
if ( guest_l4e_get_flags(new_gl4e) & _PAGE_PRESENT )
{
gfn_t gl3gfn = guest_l4e_get_gfn(new_gl4e);
- mfn_t gl3mfn = gfn_to_mfn_query(d, gl3gfn, &p2mt);
+ mfn_t gl3mfn = get_gfn_query(d, gl3gfn, &p2mt);
if ( p2m_is_ram(p2mt) )
sl3mfn = get_shadow_status(v, gl3mfn, SH_type_l3_shadow);
else if ( p2mt != p2m_populate_on_demand )
@@ -2275,6 +2275,7 @@ static int validate_gl4e(struct vcpu *v,
if ( mfn_valid(sl3mfn) )
shadow_resync_all(v);
#endif
+ put_gfn(d, gfn_x(gl3gfn));
}
l4e_propagate_from_guest(v, new_gl4e, sl3mfn, &new_sl4e, ft_prefetch);
@@ -2322,7 +2323,7 @@ static int validate_gl3e(struct vcpu *v,
if ( guest_l3e_get_flags(new_gl3e) & _PAGE_PRESENT )
{
gfn_t gl2gfn = guest_l3e_get_gfn(new_gl3e);
- mfn_t gl2mfn = gfn_to_mfn_query(v->domain, gl2gfn, &p2mt);
+ mfn_t gl2mfn = get_gfn_query(v->domain, gl2gfn, &p2mt);
if ( p2m_is_ram(p2mt) )
sl2mfn = get_shadow_status(v, gl2mfn, SH_type_l2_shadow);
else if ( p2mt != p2m_populate_on_demand )
@@ -2332,6 +2333,7 @@ static int validate_gl3e(struct vcpu *v,
if ( mfn_valid(sl2mfn) )
shadow_resync_all(v);
#endif
+ put_gfn(v->domain, gfn_x(gl2gfn));
}
l3e_propagate_from_guest(v, new_gl3e, sl2mfn, &new_sl3e, ft_prefetch);
result |= shadow_set_l3e(v, sl3p, new_sl3e, sl3mfn);
@@ -2371,11 +2373,12 @@ static int validate_gl2e(struct vcpu *v,
}
else
{
- mfn_t gl1mfn = gfn_to_mfn_query(v->domain, gl1gfn, &p2mt);
+ mfn_t gl1mfn = get_gfn_query(v->domain, gl1gfn, &p2mt);
if ( p2m_is_ram(p2mt) )
sl1mfn = get_shadow_status(v, gl1mfn, SH_type_l1_shadow);
else if ( p2mt != p2m_populate_on_demand )
result |= SHADOW_SET_ERROR;
+ put_gfn(v->domain, gfn_x(gl1gfn));
}
}
l2e_propagate_from_guest(v, new_gl2e, sl1mfn, &new_sl2e, ft_prefetch);
@@ -2441,7 +2444,7 @@ static int validate_gl1e(struct vcpu *v,
perfc_incr(shadow_validate_gl1e_calls);
gfn = guest_l1e_get_gfn(new_gl1e);
- gmfn = gfn_to_mfn_query(v->domain, gfn, &p2mt);
+ gmfn = get_gfn_query(v->domain, gfn, &p2mt);
l1e_propagate_from_guest(v, new_gl1e, gmfn, &new_sl1e, ft_prefetch, p2mt);
result |= shadow_set_l1e(v, sl1p, new_sl1e, p2mt, sl1mfn);
@@ -2463,6 +2466,7 @@ static int validate_gl1e(struct vcpu *v,
}
#endif /* OOS */
+ put_gfn(v->domain, gfn_x(gfn));
return result;
}
@@ -2501,10 +2505,11 @@ void sh_resync_l1(struct vcpu *v, mfn_t
shadow_l1e_t nsl1e;
gfn = guest_l1e_get_gfn(gl1e);
- gmfn = gfn_to_mfn_query(v->domain, gfn, &p2mt);
+ gmfn = get_gfn_query(v->domain, gfn, &p2mt);
l1e_propagate_from_guest(v, gl1e, gmfn, &nsl1e, ft_prefetch, p2mt);
rc |= shadow_set_l1e(v, sl1p, nsl1e, p2mt, sl1mfn);
+ put_gfn(v->domain, gfn_x(gfn));
*snpl1p = gl1e;
}
});
@@ -2824,7 +2829,7 @@ static void sh_prefetch(struct vcpu *v,
/* Look at the gfn that the l1e is pointing at */
gfn = guest_l1e_get_gfn(gl1e);
- gmfn = gfn_to_mfn_query(v->domain, gfn, &p2mt);
+ gmfn = get_gfn_query(v->domain, gfn, &p2mt);
/* Propagate the entry. */
l1e_propagate_from_guest(v, gl1e, gmfn, &sl1e, ft_prefetch, p2mt);
@@ -2834,6 +2839,8 @@ static void sh_prefetch(struct vcpu *v,
if ( snpl1p != NULL )
snpl1p[i] = gl1e;
#endif /* OOS */
+
+ put_gfn(v->domain, gfn_x(gfn));
}
if ( gl1p != NULL )
sh_unmap_domain_page(gl1p);
@@ -3182,7 +3189,7 @@ static int sh_page_fault(struct vcpu *v,
/* What mfn is the guest trying to access? */
gfn = guest_l1e_get_gfn(gw.l1e);
- gmfn = gfn_to_mfn_guest(d, gfn, &p2mt);
+ gmfn = get_gfn_guest(d, gfn, &p2mt);
if ( shadow_mode_refcounts(d) &&
((!p2m_is_valid(p2mt) && !p2m_is_grant(p2mt)) ||
@@ -3192,6 +3199,7 @@ static int sh_page_fault(struct vcpu *v,
SHADOW_PRINTK("BAD gfn=%"SH_PRI_gfn" gmfn=%"PRI_mfn"\n",
gfn_x(gfn), mfn_x(gmfn));
reset_early_unshadow(v);
+ put_gfn(d, gfn_x(gfn));
goto propagate;
}
@@ -3236,6 +3244,7 @@ static int sh_page_fault(struct vcpu *v,
if ( rc & GW_RMWR_REWALK )
{
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
goto rewalk;
}
#endif /* OOS */
@@ -3244,6 +3253,7 @@ static int sh_page_fault(struct vcpu *v,
{
perfc_incr(shadow_inconsistent_gwalk);
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
goto rewalk;
}
@@ -3270,6 +3280,7 @@ static int sh_page_fault(struct vcpu *v,
ASSERT(d->is_shutting_down);
#endif
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
trace_shadow_gen(TRC_SHADOW_DOMF_DYING, va);
return 0;
}
@@ -3287,6 +3298,7 @@ static int sh_page_fault(struct vcpu *v,
* failed. We cannot safely continue since some page is still
* OOS but not in the hash table anymore. */
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
return 0;
}
@@ -3296,6 +3308,7 @@ static int sh_page_fault(struct vcpu *v,
{
perfc_incr(shadow_inconsistent_gwalk);
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
goto rewalk;
}
#endif /* OOS */
@@ -3389,6 +3402,7 @@ static int sh_page_fault(struct vcpu *v,
SHADOW_PRINTK("fixed\n");
shadow_audit_tables(v);
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
return EXCRET_fault_fixed;
emulate:
@@ -3457,6 +3471,7 @@ static int sh_page_fault(struct vcpu *v,
sh_audit_gw(v, &gw);
shadow_audit_tables(v);
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
this_cpu(trace_emulate_write_val) = 0;
@@ -3595,6 +3610,7 @@ static int sh_page_fault(struct vcpu *v,
shadow_audit_tables(v);
reset_early_unshadow(v);
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
trace_shadow_gen(TRC_SHADOW_MMIO, va);
return (handle_mmio_with_translation(va, gpa >> PAGE_SHIFT)
? EXCRET_fault_fixed : 0);
@@ -3605,6 +3621,7 @@ static int sh_page_fault(struct vcpu *v,
shadow_audit_tables(v);
reset_early_unshadow(v);
paging_unlock(d);
+ put_gfn(d, gfn_x(gfn));
propagate:
trace_not_shadow_fault(gw.l1e, va);
@@ -4292,7 +4309,7 @@ sh_update_cr3(struct vcpu *v, int do_loc
if ( guest_l3e_get_flags(gl3e[i]) & _PAGE_PRESENT )
{
gl2gfn = guest_l3e_get_gfn(gl3e[i]);
- gl2mfn = gfn_to_mfn_query(d, gl2gfn, &p2mt);
+ gl2mfn = get_gfn_query_unlocked(d, gfn_x(gl2gfn), &p2mt);
if ( p2m_is_ram(p2mt) )
flush |= sh_remove_write_access(v, gl2mfn, 2, 0);
}
@@ -4305,13 +4322,14 @@ sh_update_cr3(struct vcpu *v, int do_loc
if ( guest_l3e_get_flags(gl3e[i]) & _PAGE_PRESENT )
{
gl2gfn = guest_l3e_get_gfn(gl3e[i]);
- gl2mfn = gfn_to_mfn_query(d, gl2gfn, &p2mt);
+ gl2mfn = get_gfn_query(d, gl2gfn, &p2mt);
if ( p2m_is_ram(p2mt) )
sh_set_toplevel_shadow(v, i, gl2mfn, (i == 3)
? SH_type_l2h_shadow
: SH_type_l2_shadow);
else
sh_set_toplevel_shadow(v, i, _mfn(INVALID_MFN), 0);
+ put_gfn(d, gfn_x(gl2gfn));
}
else
sh_set_toplevel_shadow(v, i, _mfn(INVALID_MFN), 0);
@@ -4689,11 +4707,12 @@ static void sh_pagetable_dying(struct vc
int flush = 0;
int fast_path = 0;
paddr_t gcr3 = 0;
- mfn_t smfn, gmfn;
p2m_type_t p2mt;
char *gl3pa = NULL;
guest_l3e_t *gl3e = NULL;
paddr_t gl2a = 0;
+ unsigned long l3gfn;
+ mfn_t l3mfn;
paging_lock(v->domain);
@@ -4702,8 +4721,9 @@ static void sh_pagetable_dying(struct vc
if ( gcr3 == gpa )
fast_path = 1;
- gmfn = gfn_to_mfn_query(v->domain, _gfn(gpa >> PAGE_SHIFT), &p2mt);
- if ( !mfn_valid(gmfn) || !p2m_is_ram(p2mt) )
+ l3gfn = gpa >> PAGE_SHIFT;
+ l3mfn = get_gfn_query(v->domain, _gfn(l3gfn), &p2mt);
+ if ( !mfn_valid(l3mfn) || !p2m_is_ram(p2mt) )
{
printk(XENLOG_DEBUG "sh_pagetable_dying: gpa not valid %"PRIpaddr"\n",
gpa);
@@ -4711,19 +4731,24 @@ static void sh_pagetable_dying(struct vc
}
if ( !fast_path )
{
- gl3pa = sh_map_domain_page(gmfn);
+ gl3pa = sh_map_domain_page(l3mfn);
gl3e = (guest_l3e_t *)(gl3pa + ((unsigned long)gpa & ~PAGE_MASK));
}
for ( i = 0; i < 4; i++ )
{
+ unsigned long gfn;
+ mfn_t smfn, gmfn;
+
if ( fast_path )
smfn = _mfn(pagetable_get_pfn(v->arch.shadow_table[i]));
else
{
/* retrieving the l2s */
gl2a = guest_l3e_get_paddr(gl3e[i]);
- gmfn = gfn_to_mfn_query(v->domain, _gfn(gl2a >> PAGE_SHIFT),
&p2mt);
+ gfn = gl2a >> PAGE_SHIFT;
+ gmfn = get_gfn_query(v->domain, _gfn(gfn), &p2mt);
smfn = shadow_hash_lookup(v, mfn_x(gmfn), SH_type_l2_pae_shadow);
+ put_gfn(v->domain, gfn);
}
if ( mfn_valid(smfn) )
@@ -4747,6 +4772,7 @@ static void sh_pagetable_dying(struct vc
out:
if ( !fast_path )
unmap_domain_page(gl3pa);
+ put_gfn(v->domain, l3gfn);
paging_unlock(v->domain);
}
#else
@@ -4757,12 +4783,14 @@ static void sh_pagetable_dying(struct vc
paging_lock(v->domain);
- gmfn = gfn_to_mfn_query(v->domain, _gfn(gpa >> PAGE_SHIFT), &p2mt);
+ gmfn = get_gfn_query(v->domain, _gfn(gpa >> PAGE_SHIFT), &p2mt);
#if GUEST_PAGING_LEVELS == 2
smfn = shadow_hash_lookup(v, mfn_x(gmfn), SH_type_l2_32_shadow);
#else
smfn = shadow_hash_lookup(v, mfn_x(gmfn), SH_type_l4_64_shadow);
#endif
+ put_gfn(v->domain, gpa >> PAGE_SHIFT);
+
if ( mfn_valid(smfn) )
{
mfn_to_page(gmfn)->shadow_flags |= SHF_pagetable_dying;
@@ -4811,15 +4839,22 @@ static mfn_t emulate_gva_to_mfn(struct v
/* Translate the GFN to an MFN */
ASSERT(!paging_locked_by_me(v->domain));
- mfn = gfn_to_mfn_guest(v->domain, _gfn(gfn), &p2mt);
+ mfn = get_gfn_guest(v->domain, _gfn(gfn), &p2mt);
if ( p2m_is_readonly(p2mt) )
+ {
+ put_gfn(v->domain, gfn);
return _mfn(READONLY_GFN);
+ }
if ( !p2m_is_ram(p2mt) )
+ {
+ put_gfn(v->domain, gfn);
return _mfn(BAD_GFN_TO_MFN);
+ }
ASSERT(mfn_valid(mfn));
v->arch.paging.last_write_was_pt = !!sh_mfn_is_a_page_table(mfn);
+ put_gfn(v->domain, gfn);
return mfn;
}
@@ -5220,7 +5255,7 @@ int sh_audit_l1_table(struct vcpu *v, mf
{
gfn = guest_l1e_get_gfn(*gl1e);
mfn = shadow_l1e_get_mfn(*sl1e);
- gmfn = gfn_to_mfn_query(v->domain, gfn, &p2mt);
+ gmfn = get_gfn_query_unlocked(v->domain, gfn_x(gfn), &p2mt);
if ( !p2m_is_grant(p2mt) && mfn_x(gmfn) != mfn_x(mfn) )
AUDIT_FAIL(1, "bad translation: gfn %" SH_PRI_gfn
" --> %" PRI_mfn " != mfn %" PRI_mfn,
@@ -5291,16 +5326,17 @@ int sh_audit_l2_table(struct vcpu *v, mf
mfn = shadow_l2e_get_mfn(*sl2e);
gmfn = (guest_l2e_get_flags(*gl2e) & _PAGE_PSE)
? get_fl1_shadow_status(v, gfn)
- : get_shadow_status(v, gfn_to_mfn_query(v->domain, gfn, &p2mt),
- SH_type_l1_shadow);
+ : get_shadow_status(v,
+ get_gfn_query_unlocked(v->domain, gfn_x(gfn),
+ &p2mt), SH_type_l1_shadow);
if ( mfn_x(gmfn) != mfn_x(mfn) )
AUDIT_FAIL(2, "bad translation: gfn %" SH_PRI_gfn
" (--> %" PRI_mfn ")"
" --> %" PRI_mfn " != mfn %" PRI_mfn,
gfn_x(gfn),
(guest_l2e_get_flags(*gl2e) & _PAGE_PSE) ? 0
- : mfn_x(gfn_to_mfn_query(v->domain,
- gfn, &p2mt)), mfn_x(gmfn), mfn_x(mfn));
+ : mfn_x(get_gfn_query_unlocked(v->domain,
+ gfn_x(gfn), &p2mt)), mfn_x(gmfn),
mfn_x(mfn));
}
});
sh_unmap_domain_page(gp);
@@ -5339,7 +5375,8 @@ int sh_audit_l3_table(struct vcpu *v, mf
{
gfn = guest_l3e_get_gfn(*gl3e);
mfn = shadow_l3e_get_mfn(*sl3e);
- gmfn = get_shadow_status(v, gfn_to_mfn_query(v->domain, gfn,
&p2mt),
+ gmfn = get_shadow_status(v, get_gfn_query_unlocked(
+ v->domain, gfn_x(gfn), &p2mt),
((GUEST_PAGING_LEVELS == 3 ||
is_pv_32on64_vcpu(v))
&& !shadow_mode_external(v->domain)
@@ -5387,8 +5424,8 @@ int sh_audit_l4_table(struct vcpu *v, mf
{
gfn = guest_l4e_get_gfn(*gl4e);
mfn = shadow_l4e_get_mfn(*sl4e);
- gmfn = get_shadow_status(v, gfn_to_mfn_query(v->domain,
- gfn, &p2mt),
+ gmfn = get_shadow_status(v, get_gfn_query_unlocked(
+ v->domain, gfn_x(gfn), &p2mt),
SH_type_l3_shadow);
if ( mfn_x(gmfn) != mfn_x(mfn) )
AUDIT_FAIL(4, "bad translation: gfn %" SH_PRI_gfn
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/mm/shadow/types.h
--- a/xen/arch/x86/mm/shadow/types.h
+++ b/xen/arch/x86/mm/shadow/types.h
@@ -191,11 +191,11 @@ static inline shadow_l4e_t shadow_l4e_fr
})
#endif
- /* Override gfn_to_mfn to work with gfn_t */
-#undef gfn_to_mfn_query
-#define gfn_to_mfn_query(d, g, t) gfn_to_mfn_type((d), gfn_x(g), (t),
p2m_query)
-#undef gfn_to_mfn_guest
-#define gfn_to_mfn_guest(d, g, t) gfn_to_mfn_type((d), gfn_x(g), (t),
p2m_guest)
+ /* Override get_gfn to work with gfn_t */
+#undef get_gfn_query
+#define get_gfn_query(d, g, t) get_gfn_type((d), gfn_x(g), (t), p2m_query)
+#undef get_gfn_guest
+#define get_gfn_guest(d, g, t) get_gfn_type((d), gfn_x(g), (t), p2m_guest)
/* The shadow types needed for the various levels. */
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/physdev.c
--- a/xen/arch/x86/physdev.c
+++ b/xen/arch/x86/physdev.c
@@ -297,16 +297,20 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_H
break;
ret = -EINVAL;
- mfn = gmfn_to_mfn(current->domain, info.gmfn);
+ mfn = get_gfn_untyped(current->domain, info.gmfn);
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), v->domain,
PGT_writable_page) )
+ {
+ put_gfn(current->domain, info.gmfn);
break;
+ }
if ( cmpxchg(&v->domain->arch.pv_domain.pirq_eoi_map_mfn,
0, mfn) != 0 )
{
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(current->domain, info.gmfn);
ret = -EBUSY;
break;
}
@@ -316,10 +320,12 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_H
{
v->domain->arch.pv_domain.pirq_eoi_map_mfn = 0;
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(current->domain, info.gmfn);
ret = -ENOSPC;
break;
}
+ put_gfn(current->domain, info.gmfn);
ret = 0;
break;
}
diff -r 4699decb8424 -r 7461834954c8 xen/arch/x86/traps.c
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -673,11 +673,12 @@ int wrmsr_hypervisor_regs(uint32_t idx,
return 0;
}
- mfn = gmfn_to_mfn(d, gmfn);
+ mfn = get_gfn_untyped(d, gmfn);
if ( !mfn_valid(mfn) ||
!get_page_and_type(mfn_to_page(mfn), d, PGT_writable_page) )
{
+ put_gfn(d, gmfn);
gdprintk(XENLOG_WARNING,
"Bad GMFN %lx (MFN %lx) to MSR %08x\n",
gmfn, mfn, base + idx);
@@ -689,6 +690,7 @@ int wrmsr_hypervisor_regs(uint32_t idx,
unmap_domain_page(hypercall_page);
put_page_and_type(mfn_to_page(mfn));
+ put_gfn(d, gmfn);
break;
}
@@ -2347,18 +2349,25 @@ static int emulate_privileged_op(struct
arch_set_cr2(v, *reg);
break;
- case 3: /* Write CR3 */
+ case 3: {/* Write CR3 */
+ unsigned long mfn, gfn;
domain_lock(v->domain);
if ( !is_pv_32on64_vcpu(v) )
- rc = new_guest_cr3(gmfn_to_mfn(v->domain,
xen_cr3_to_pfn(*reg)));
+ {
+ gfn = xen_cr3_to_pfn(*reg);
#ifdef CONFIG_COMPAT
- else
- rc = new_guest_cr3(gmfn_to_mfn(v->domain,
compat_cr3_to_pfn(*reg)));
+ } else {
+ gfn = compat_cr3_to_pfn(*reg);
#endif
+ }
+ mfn = get_gfn_untyped(v->domain, gfn);
+ rc = new_guest_cr3(mfn);
+ put_gfn(v->domain, gfn);
domain_unlock(v->domain);
if ( rc == 0 ) /* not okay */
goto fail;
break;
+ }
case 4: /* Write CR4 */
v->arch.pv_vcpu.ctrlreg[4] = pv_guest_cr4_fixup(v, *reg);
diff -r 4699decb8424 -r 7461834954c8 xen/common/grant_table.c
--- a/xen/common/grant_table.c
+++ b/xen/common/grant_table.c
@@ -110,7 +110,7 @@ static unsigned inline int max_nr_maptra
#define gfn_to_mfn_private(_d, _gfn) ({ \
p2m_type_t __p2mt; \
unsigned long __x; \
- __x = mfn_x(gfn_to_mfn_unshare((_d), (_gfn), &__p2mt)); \
+ __x = mfn_x(get_gfn_unshare((_d), (_gfn), &__p2mt)); \
BUG_ON(p2m_is_shared(__p2mt)); /* XXX fixme */ \
if ( !p2m_is_valid(__p2mt) ) \
__x = INVALID_MFN; \
@@ -150,10 +150,10 @@ static int __get_paged_frame(unsigned lo
mfn_t mfn;
if ( readonly )
- mfn = gfn_to_mfn(rd, gfn, &p2mt);
+ mfn = get_gfn(rd, gfn, &p2mt);
else
{
- mfn = gfn_to_mfn_unshare(rd, gfn, &p2mt);
+ mfn = get_gfn_unshare(rd, gfn, &p2mt);
BUG_ON(p2m_is_shared(p2mt));
/* XXX Here, and above in gfn_to_mfn_private, need to handle
* XXX failure to unshare. */
@@ -164,14 +164,16 @@ static int __get_paged_frame(unsigned lo
if ( p2m_is_paging(p2mt) )
{
p2m_mem_paging_populate(rd, gfn);
+ put_gfn(rd, gfn);
rc = GNTST_eagain;
}
} else {
+ put_gfn(rd, gfn);
*frame = INVALID_MFN;
rc = GNTST_bad_page;
}
#else
- *frame = readonly ? gmfn_to_mfn(rd, gfn) : gfn_to_mfn_private(rd, gfn);
+ *frame = readonly ? get_gfn_untyped(rd, gfn) : gfn_to_mfn_private(rd, gfn);
#endif
return rc;
@@ -468,13 +470,14 @@ __gnttab_map_grant_ref(
struct domain *ld, *rd, *owner;
struct vcpu *led;
int handle;
+ unsigned long gfn = INVALID_GFN;
unsigned long frame = 0, nr_gets = 0;
struct page_info *pg;
int rc = GNTST_okay;
u32 old_pin;
u32 act_pin;
unsigned int cache_flags;
- struct active_grant_entry *act;
+ struct active_grant_entry *act = NULL;
struct grant_mapping *mt;
grant_entry_v1_t *sha1;
grant_entry_v2_t *sha2;
@@ -565,7 +568,6 @@ __gnttab_map_grant_ref(
if ( !act->pin )
{
- unsigned long gfn;
unsigned long frame;
gfn = sha1 ? sha1->frame : sha2->full_page.frame;
@@ -698,6 +700,7 @@ __gnttab_map_grant_ref(
op->handle = handle;
op->status = GNTST_okay;
+ put_gfn(rd, gfn);
rcu_unlock_domain(rd);
return;
@@ -735,6 +738,8 @@ __gnttab_map_grant_ref(
gnttab_clear_flag(_GTF_reading, status);
unlock_out:
+ if ( gfn != INVALID_GFN )
+ put_gfn(rd, gfn);
spin_unlock(&rd->grant_table->lock);
op->status = rc;
put_maptrack_handle(ld->grant_table, handle);
@@ -1475,6 +1480,7 @@ gnttab_transfer(
/* Check the passed page frame for basic validity. */
if ( unlikely(!mfn_valid(mfn)) )
{
+ put_gfn(d, gop.mfn);
gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
(unsigned long)gop.mfn);
gop.status = GNTST_bad_page;
@@ -1484,6 +1490,7 @@ gnttab_transfer(
page = mfn_to_page(mfn);
if ( unlikely(is_xen_heap_page(page)) )
{
+ put_gfn(d, gop.mfn);
gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
(unsigned long)gop.mfn);
gop.status = GNTST_bad_page;
@@ -1492,6 +1499,7 @@ gnttab_transfer(
if ( steal_page(d, page, 0) < 0 )
{
+ put_gfn(d, gop.mfn);
gop.status = GNTST_bad_page;
goto copyback;
}
@@ -1504,6 +1512,7 @@ gnttab_transfer(
/* Find the target domain. */
if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) )
{
+ put_gfn(d, gop.mfn);
gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
gop.domid);
page->count_info &= ~(PGC_count_mask|PGC_allocated);
@@ -1514,6 +1523,7 @@ gnttab_transfer(
if ( xsm_grant_transfer(d, e) )
{
+ put_gfn(d, gop.mfn);
gop.status = GNTST_permission_denied;
unlock_and_copyback:
rcu_unlock_domain(e);
@@ -1566,6 +1576,7 @@ gnttab_transfer(
e->tot_pages, e->max_pages, gop.ref, e->is_dying);
spin_unlock(&e->page_alloc_lock);
rcu_unlock_domain(e);
+ put_gfn(d, gop.mfn);
page->count_info &= ~(PGC_count_mask|PGC_allocated);
free_domheap_page(page);
gop.status = GNTST_general_error;
@@ -1579,6 +1590,7 @@ gnttab_transfer(
page_set_owner(page, e);
spin_unlock(&e->page_alloc_lock);
+ put_gfn(d, gop.mfn);
TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
@@ -1850,6 +1862,8 @@ __acquire_grant_for_copy(
{
gfn = sha1->frame;
rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
+ /* We drop this immediately per the comments at the top */
+ put_gfn(rd, gfn);
if ( rc != GNTST_okay )
goto unlock_out;
act->gfn = gfn;
@@ -1862,6 +1876,7 @@ __acquire_grant_for_copy(
{
gfn = sha2->full_page.frame;
rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
+ put_gfn(rd, gfn);
if ( rc != GNTST_okay )
goto unlock_out;
act->gfn = gfn;
@@ -1874,6 +1889,7 @@ __acquire_grant_for_copy(
{
gfn = sha2->sub_page.frame;
rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
+ put_gfn(rd, gfn);
if ( rc != GNTST_okay )
goto unlock_out;
act->gfn = gfn;
@@ -1973,6 +1989,7 @@ __gnttab_copy(
{
#ifdef CONFIG_X86
rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
+ put_gfn(sd, op->source.u.gmfn);
if ( rc != GNTST_okay )
goto error_out;
#else
@@ -2012,6 +2029,7 @@ __gnttab_copy(
{
#ifdef CONFIG_X86
rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
+ put_gfn(dd, op->dest.u.gmfn);
if ( rc != GNTST_okay )
goto error_out;
#else
diff -r 4699decb8424 -r 7461834954c8 xen/common/memory.c
--- a/xen/common/memory.c
+++ b/xen/common/memory.c
@@ -162,11 +162,12 @@ int guest_remove_page(struct domain *d,
unsigned long mfn;
#ifdef CONFIG_X86
- mfn = mfn_x(gfn_to_mfn(d, gmfn, &p2mt));
+ mfn = mfn_x(get_gfn(d, gmfn, &p2mt));
if ( unlikely(p2m_is_paging(p2mt)) )
{
guest_physmap_remove_page(d, gmfn, mfn, PAGE_ORDER_4K);
p2m_mem_paging_drop_page(d, gmfn);
+ put_gfn(d, gmfn);
return 1;
}
#else
@@ -174,6 +175,7 @@ int guest_remove_page(struct domain *d,
#endif
if ( unlikely(!mfn_valid(mfn)) )
{
+ put_gfn(d, gmfn);
gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
d->domain_id, gmfn);
return 0;
@@ -187,12 +189,14 @@ int guest_remove_page(struct domain *d,
{
put_page_and_type(page);
guest_physmap_remove_page(d, gmfn, mfn, PAGE_ORDER_4K);
+ put_gfn(d, gmfn);
return 1;
}
#endif /* CONFIG_X86 */
if ( unlikely(!get_page(page, d)) )
{
+ put_gfn(d, gmfn);
gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
return 0;
}
@@ -206,6 +210,7 @@ int guest_remove_page(struct domain *d,
guest_physmap_remove_page(d, gmfn, mfn, PAGE_ORDER_4K);
put_page(page);
+ put_gfn(d, gmfn);
return 1;
}
@@ -265,7 +270,7 @@ static long memory_exchange(XEN_GUEST_HA
PAGE_LIST_HEAD(out_chunk_list);
unsigned long in_chunk_order, out_chunk_order;
xen_pfn_t gpfn, gmfn, mfn;
- unsigned long i, j, k;
+ unsigned long i, j, k = 0; /* gcc ... */
unsigned int memflags = 0;
long rc = 0;
struct domain *d;
@@ -363,9 +368,10 @@ static long memory_exchange(XEN_GUEST_HA
p2m_type_t p2mt;
/* Shared pages cannot be exchanged */
- mfn = mfn_x(gfn_to_mfn_unshare(d, gmfn + k, &p2mt));
+ mfn = mfn_x(get_gfn_unshare(d, gmfn + k, &p2mt));
if ( p2m_is_shared(p2mt) )
{
+ put_gfn(d, gmfn + k);
rc = -ENOMEM;
goto fail;
}
@@ -374,6 +380,7 @@ static long memory_exchange(XEN_GUEST_HA
#endif
if ( unlikely(!mfn_valid(mfn)) )
{
+ put_gfn(d, gmfn + k);
rc = -EINVAL;
goto fail;
}
@@ -382,11 +389,13 @@ static long memory_exchange(XEN_GUEST_HA
if ( unlikely(steal_page(d, page, MEMF_no_refcount)) )
{
+ put_gfn(d, gmfn + k);
rc = -EINVAL;
goto fail;
}
page_list_add(page, &in_chunk_list);
+ put_gfn(d, gmfn + k);
}
}
@@ -487,8 +496,12 @@ static long memory_exchange(XEN_GUEST_HA
fail:
/* Reassign any input pages we managed to steal. */
while ( (page = page_list_remove_head(&in_chunk_list)) )
+ {
+ put_gfn(d, gmfn + k--);
if ( assign_pages(d, page, 0, MEMF_no_refcount) )
BUG();
+ }
+
dying:
rcu_unlock_domain(d);
/* Free any output pages we managed to allocate. */
diff -r 4699decb8424 -r 7461834954c8 xen/common/tmem_xen.c
--- a/xen/common/tmem_xen.c
+++ b/xen/common/tmem_xen.c
@@ -109,22 +109,28 @@ static inline void *cli_get_page(tmem_cl
struct page_info *page;
int ret;
- cli_mfn = mfn_x(gfn_to_mfn(current->domain, cmfn, &t));
+ cli_mfn = mfn_x(get_gfn(current->domain, cmfn, &t));
if ( t != p2m_ram_rw || !mfn_valid(cli_mfn) )
+ {
+ put_gfn(current->domain, (unsigned long) cmfn);
return NULL;
+ }
page = mfn_to_page(cli_mfn);
if ( cli_write )
ret = get_page_and_type(page, current->domain, PGT_writable_page);
else
ret = get_page(page, current->domain);
if ( !ret )
+ {
+ put_gfn(current->domain, (unsigned long) cmfn);
return NULL;
+ }
*pcli_mfn = cli_mfn;
*pcli_pfp = (pfp_t *)page;
return map_domain_page(cli_mfn);
}
-static inline void cli_put_page(void *cli_va, pfp_t *cli_pfp,
+static inline void cli_put_page(tmem_cli_mfn_t cmfn, void *cli_va, pfp_t
*cli_pfp,
unsigned long cli_mfn, bool_t mark_dirty)
{
if ( mark_dirty )
@@ -135,6 +141,7 @@ static inline void cli_put_page(void *cl
else
put_page((struct page_info *)cli_pfp);
unmap_domain_page(cli_va);
+ put_gfn(current->domain, (unsigned long) cmfn);
}
#endif
@@ -169,7 +176,7 @@ EXPORT int tmh_copy_from_client(pfp_t *p
(pfn_offset+len <= PAGE_SIZE) )
memcpy((char *)tmem_va+tmem_offset,(char *)cli_va+pfn_offset,len);
if ( !tmemc )
- cli_put_page(cli_va, cli_pfp, cli_mfn, 0);
+ cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 0);
unmap_domain_page(tmem_va);
return 1;
}
@@ -197,7 +204,7 @@ EXPORT int tmh_compress_from_client(tmem
ASSERT(ret == LZO_E_OK);
*out_va = dmem;
if ( !tmemc )
- cli_put_page(cli_va, cli_pfp, cli_mfn, 0);
+ cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 0);
unmap_domain_page(cli_va);
return 1;
}
@@ -225,7 +232,7 @@ EXPORT int tmh_copy_to_client(tmem_cli_m
memcpy((char *)cli_va+pfn_offset,(char *)tmem_va+tmem_offset,len);
unmap_domain_page(tmem_va);
if ( !tmemc )
- cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
+ cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 1);
mb();
return 1;
}
@@ -249,7 +256,7 @@ EXPORT int tmh_decompress_to_client(tmem
ASSERT(ret == LZO_E_OK);
ASSERT(out_len == PAGE_SIZE);
if ( !tmemc )
- cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
+ cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 1);
mb();
return 1;
}
@@ -271,7 +278,7 @@ EXPORT int tmh_copy_tze_to_client(tmem_c
memcpy((char *)cli_va,(char *)tmem_va,len);
if ( len < PAGE_SIZE )
memset((char *)cli_va+len,0,PAGE_SIZE-len);
- cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
+ cli_put_page(cmfn, cli_va, cli_pfp, cli_mfn, 1);
mb();
return 1;
}
diff -r 4699decb8424 -r 7461834954c8 xen/include/asm-ia64/mm.h
--- a/xen/include/asm-ia64/mm.h
+++ b/xen/include/asm-ia64/mm.h
@@ -549,6 +549,8 @@ extern u64 translate_domain_pte(u64 ptev
#define gmfn_to_mfn(_d, gpfn) \
gmfn_to_mfn_foreign((_d), (gpfn))
+#define put_gfn(d, g) ((void)0)
+
#define __gpfn_invalid(_d, gpfn) \
(lookup_domain_mpa((_d), ((gpfn)<<PAGE_SHIFT), NULL) == INVALID_MFN)
diff -r 4699decb8424 -r 7461834954c8 xen/include/asm-x86/guest_pt.h
--- a/xen/include/asm-x86/guest_pt.h
+++ b/xen/include/asm-x86/guest_pt.h
@@ -51,9 +51,9 @@ gfn_to_paddr(gfn_t gfn)
return ((paddr_t)gfn_x(gfn)) << PAGE_SHIFT;
}
-/* Override gfn_to_mfn to work with gfn_t */
-#undef gfn_to_mfn
-#define gfn_to_mfn(d, g, t) gfn_to_mfn_type((d), gfn_x(g), (t), p2m_alloc)
+/* Override get_gfn to work with gfn_t */
+#undef get_gfn
+#define get_gfn(d, g, t) get_gfn_type((d), gfn_x(g), (t), p2m_alloc)
/* Types of the guest's page tables and access functions for them */
diff -r 4699decb8424 -r 7461834954c8 xen/include/asm-x86/p2m.h
--- a/xen/include/asm-x86/p2m.h
+++ b/xen/include/asm-x86/p2m.h
@@ -3,6 +3,7 @@
*
* physical-to-machine mappings for automatically-translated domains.
*
+ * Copyright (c) 2011 GridCentric Inc. (Andres Lagar-Cavilla)
* Copyright (c) 2007 Advanced Micro Devices (Wei Huang)
* Parts of this code are Copyright (c) 2006-2007 by XenSource Inc.
* Parts of this code are Copyright (c) 2006 by Michael A Fetterman
@@ -305,47 +306,66 @@ struct p2m_domain *p2m_get_p2m(struct vc
#define p2m_get_pagetable(p2m) ((p2m)->phys_table)
+/**** p2m query accessors. After calling any of the variants below, you
+ * need to call put_gfn(domain, gfn). If you don't, you'll lock the
+ * hypervisor. ****/
/* Read a particular P2M table, mapping pages as we go. Most callers
- * should _not_ call this directly; use the other gfn_to_mfn_* functions
+ * should _not_ call this directly; use the other get_gfn* functions
* below unless you know you want to walk a p2m that isn't a domain's
* main one.
* If the lookup succeeds, the return value is != INVALID_MFN and
* *page_order is filled in with the order of the superpage (if any) that
* the entry was found in. */
-mfn_t gfn_to_mfn_type_p2m(struct p2m_domain *p2m, unsigned long gfn,
+mfn_t get_gfn_type_access(struct p2m_domain *p2m, unsigned long gfn,
p2m_type_t *t, p2m_access_t *a, p2m_query_t q,
unsigned int *page_order);
/* General conversion function from gfn to mfn */
-static inline mfn_t gfn_to_mfn_type(struct domain *d,
+static inline mfn_t get_gfn_type(struct domain *d,
unsigned long gfn, p2m_type_t *t,
p2m_query_t q)
{
p2m_access_t a;
- return gfn_to_mfn_type_p2m(p2m_get_hostp2m(d), gfn, t, &a, q, NULL);
+ return get_gfn_type_access(p2m_get_hostp2m(d), gfn, t, &a, q, NULL);
}
/* Syntactic sugar: most callers will use one of these.
- * N.B. gfn_to_mfn_query() is the _only_ one guaranteed not to take the
+ * N.B. get_gfn_query() is the _only_ one guaranteed not to take the
* p2m lock; none of the others can be called with the p2m or paging
* lock held. */
-#define gfn_to_mfn(d, g, t) gfn_to_mfn_type((d), (g), (t), p2m_alloc)
-#define gfn_to_mfn_query(d, g, t) gfn_to_mfn_type((d), (g), (t), p2m_query)
-#define gfn_to_mfn_guest(d, g, t) gfn_to_mfn_type((d), (g), (t), p2m_guest)
-#define gfn_to_mfn_unshare(d, g, t) gfn_to_mfn_type((d), (g), (t), p2m_unshare)
+#define get_gfn(d, g, t) get_gfn_type((d), (g), (t), p2m_alloc)
+#define get_gfn_query(d, g, t) get_gfn_type((d), (g), (t), p2m_query)
+#define get_gfn_guest(d, g, t) get_gfn_type((d), (g), (t), p2m_guest)
+#define get_gfn_unshare(d, g, t) get_gfn_type((d), (g), (t), p2m_unshare)
/* Compatibility function exporting the old untyped interface */
-static inline unsigned long gmfn_to_mfn(struct domain *d, unsigned long gpfn)
+static inline unsigned long get_gfn_untyped(struct domain *d, unsigned long
gpfn)
{
mfn_t mfn;
p2m_type_t t;
- mfn = gfn_to_mfn(d, gpfn, &t);
+ mfn = get_gfn(d, gpfn, &t);
if ( p2m_is_valid(t) )
return mfn_x(mfn);
return INVALID_MFN;
}
+/* This is a noop for now. */
+static inline void __put_gfn(struct p2m_domain *p2m, unsigned long gfn)
+{
+}
+
+#define put_gfn(d, gfn) __put_gfn(p2m_get_hostp2m((d)), (gfn))
+
+/* These are identical for now. The intent is to have the caller not worry
+ * about put_gfn. To only be used in printk's, crash situations, or to
+ * peek at a type. You're not holding the p2m entry exclsively after calling
+ * this. */
+#define get_gfn_unlocked(d, g, t) get_gfn_type((d), (g), (t),
p2m_alloc)
+#define get_gfn_query_unlocked(d, g, t) get_gfn_type((d), (g), (t),
p2m_query)
+#define get_gfn_guest_unlocked(d, g, t) get_gfn_type((d), (g), (t),
p2m_guest)
+#define get_gfn_unshare_unlocked(d, g, t) get_gfn_type((d), (g), (t),
p2m_unshare)
+
/* General conversion function from mfn to gfn */
static inline unsigned long mfn_to_gfn(struct domain *d, mfn_t mfn)
{
@@ -529,7 +549,8 @@ static inline int p2m_gfn_check_limit(
#define p2m_gfn_check_limit(d, g, o) 0
#endif
-/* Directly set a p2m entry: only for use by p2m code */
+/* Directly set a p2m entry: only for use by p2m code. Does not need
+ * a call to put_gfn afterwards/ */
int set_p2m_entry(struct p2m_domain *p2m, unsigned long gfn, mfn_t mfn,
unsigned int page_order, p2m_type_t p2mt, p2m_access_t p2ma);
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home