[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Xen-devel] [patch 05/28]xen: Xen virtual mmu



Xen pagetable handling, including the machinery to implement direct
pagetables.  This means that pte update operations are intercepted so
that pseudo-physical addresses can be converted to machine addresses.
It also implements late pinning/early unpinning so that pagetables are
initialized while they're normal read-write memory as much as
possible, yet pinned to make cr3 reloads as efficient as possible.

Signed-off-by: Jeremy Fitzhardinge <jeremy@xxxxxxxxxxxxx>
Signed-off-by: Chris Wright <chrisw@xxxxxxxxxxxx>

---
 arch/i386/xen/Makefile    |    2 
 arch/i386/xen/enlighten.c |   30 ++-
 arch/i386/xen/mmu.c       |  419 +++++++++++++++++++++++++++++++++++++++++++++
 arch/i386/xen/mmu.h       |   47 +++++
 4 files changed, 493 insertions(+), 5 deletions(-)

===================================================================
--- a/arch/i386/xen/Makefile
+++ b/arch/i386/xen/Makefile
@@ -1,1 +1,1 @@ obj-y           := enlighten.o setup.o features.o
-obj-y          := enlighten.o setup.o features.o multicalls.o
+obj-y          := enlighten.o setup.o features.o multicalls.o mmu.o
===================================================================
--- a/arch/i386/xen/enlighten.c
+++ b/arch/i386/xen/enlighten.c
@@ -39,6 +39,7 @@
 #include <asm/pgtable.h>
 
 #include "xen-ops.h"
+#include "mmu.h"
 #include "multicalls.h"
 
 EXPORT_SYMBOL_GPL(hypercall_page);
@@ -565,11 +566,9 @@ static __init void xen_pagetable_setup_d
                 * Should be set_fixmap(), but shared_info is a machine
                 * address with no corresponding pseudo-phys address.
                 */
-#if 0
                set_pte_mfn(fix_to_virt(FIX_PARAVIRT_BOOTMAP),
                            PFN_DOWN(xen_start_info->shared_info),
                            PAGE_KERNEL);
-#endif
 
                HYPERVISOR_shared_info =
                        (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
@@ -578,9 +577,7 @@ static __init void xen_pagetable_setup_d
                HYPERVISOR_shared_info =
                        (struct shared_info *)__va(xen_start_info->shared_info);
 
-#if 0
        xen_pgd_pin(base);
-#endif
 
        xen_vcpu_setup(smp_processor_id());
 }
@@ -682,6 +679,31 @@ static const struct paravirt_ops xen_par
        .release_pd = xen_release_pd,
        .release_pt = xen_release_pt,
 
+       .set_pte = xen_set_pte,
+       .set_pte_at = xen_set_pte_at,
+       .set_pmd = xen_set_pmd,
+
+       .pte_val = xen_pte_val,
+       .pgd_val = xen_pgd_val,
+
+       .make_pte = xen_make_pte,
+       .make_pgd = xen_make_pgd,
+
+#ifdef CONFIG_X86_PAE
+       .set_pte_atomic = xen_set_pte_atomic,
+       .set_pte_present = xen_set_pte_at,
+       .set_pud = xen_set_pud,
+       .pte_clear = xen_pte_clear,
+       .pmd_clear = xen_pmd_clear,
+
+       .make_pmd = xen_make_pmd,
+       .pmd_val = xen_pmd_val,
+#endif /* PAE */
+
+       .activate_mm = xen_activate_mm,
+       .dup_mmap = xen_dup_mmap,
+       .exit_mmap = xen_exit_mmap,
+
        .set_lazy_mode = xen_set_lazy_mode,
 };
 
===================================================================
--- /dev/null
+++ b/arch/i386/xen/mmu.c
@@ -0,0 +1,419 @@
+/*
+ * Xen mmu operations
+ *
+ * This file contains the various mmu fetch and update operations.
+ * The most important job they must perform is the mapping between the
+ * domain's pfn and the overall machine mfns.
+ *
+ * Xen allows guests to directly update the pagetable, in a controlled
+ * fashion.  In other words, the guest modifies the same pagetable
+ * that the CPU actually uses, which eliminates the overhead of having
+ * a separate shadow pagetable.
+ *
+ * In order to allow this, it falls on the guest domain to map its
+ * notion of a "physical" pfn - which is just a domain-local linear
+ * address - into a real "machine address" which the CPU's MMU can
+ * use.
+ *
+ * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
+ * inserted directly into the pagetable.  When creating a new
+ * pte/pmd/pgd, it converts the passed pfn into an mfn.  Conversely,
+ * when reading the content back with __(pgd|pmd|pte)_val, it converts
+ * the mfn back into a pfn.
+ *
+ * The other constraint is that all pages which make up a pagetable
+ * must be mapped read-only in the guest.  This prevents uncontrolled
+ * guest updates to the pagetable.  Xen strictly enforces this, and
+ * will disallow any pagetable update which will end up mapping a
+ * pagetable page RW, and will disallow using any writable page as a
+ * pagetable.
+ *
+ * Naively, when loading %cr3 with the base of a new pagetable, Xen
+ * would need to validate the whole pagetable before going on.
+ * Naturally, this is quite slow.  The solution is to "pin" a
+ * pagetable, which enforces all the constraints on the pagetable even
+ * when it is not actively in use.  This menas that Xen can be assured
+ * that it is still valid when you do load it into %cr3, and doesn't
+ * need to revalidate it.
+ *
+ * Jeremy Fitzhardinge <jeremy@xxxxxxxxxxxxx>, XenSource Inc, 2007
+ */
+#include <asm/bug.h>
+
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
+
+#include <asm/xen/hypercall.h>
+#include <asm/paravirt.h>
+
+#include <xen/page.h>
+#include <xen/interface/xen.h>
+
+#include "mmu.h"
+
+xmaddr_t arbitrary_virt_to_machine(unsigned long address)
+{
+       pte_t *pte = lookup_address(address);
+       unsigned offset = address & PAGE_MASK;
+
+       BUG_ON(pte == NULL);
+
+       return XMADDR((pte_mfn(*pte) << PAGE_SHIFT) + offset);
+}
+
+void make_lowmem_page_readonly(void *vaddr)
+{
+       pte_t *pte, ptev;
+       unsigned long address = (unsigned long)vaddr;
+
+       pte = lookup_address(address);
+       BUG_ON(pte == NULL);
+
+       ptev = pte_wrprotect(*pte);
+
+       if(HYPERVISOR_update_va_mapping(address, ptev, 0))
+               BUG();
+}
+
+void make_lowmem_page_readwrite(void *vaddr)
+{
+       pte_t *pte, ptev;
+       unsigned long address = (unsigned long)vaddr;
+
+       pte = lookup_address(address);
+       BUG_ON(pte == NULL);
+
+       ptev = pte_mkwrite(*pte);
+
+       if(HYPERVISOR_update_va_mapping(address, ptev, 0))
+               BUG();
+}
+
+
+void xen_set_pte(pte_t *ptep, pte_t pte)
+{
+       struct mmu_update u;
+
+       u.ptr = virt_to_machine(ptep).maddr;
+       u.val = pte_val_ma(pte);
+       if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0)
+               BUG();
+}
+
+void xen_set_pmd(pmd_t *ptr, pmd_t val)
+{
+       struct mmu_update u;
+
+       u.ptr = virt_to_machine(ptr).maddr;
+       u.val = pmd_val_ma(val);
+       if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0)
+               BUG();
+}
+
+#ifdef CONFIG_X86_PAE
+void xen_set_pud(pud_t *ptr, pud_t val)
+{
+       struct mmu_update u;
+
+       u.ptr = virt_to_machine(ptr).maddr;
+       u.val = pud_val_ma(val);
+       if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0)
+               BUG();
+}
+#endif
+
+/*
+ * Associate a virtual page frame with a given physical page frame
+ * and protection flags for that frame.
+ */
+void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
+{
+       pgd_t *pgd;
+       pud_t *pud;
+       pmd_t *pmd;
+       pte_t *pte;
+
+       pgd = swapper_pg_dir + pgd_index(vaddr);
+       if (pgd_none(*pgd)) {
+               BUG();
+               return;
+       }
+       pud = pud_offset(pgd, vaddr);
+       if (pud_none(*pud)) {
+               BUG();
+               return;
+       }
+       pmd = pmd_offset(pud, vaddr);
+       if (pmd_none(*pmd)) {
+               BUG();
+               return;
+       }
+       pte = pte_offset_kernel(pmd, vaddr);
+       /* <mfn,flags> stored as-is, to permit clearing entries */
+       xen_set_pte(pte, mfn_pte(mfn, flags));
+
+       /*
+        * It's enough to flush this one mapping.
+        * (PGE mappings get flushed as well)
+        */
+       __flush_tlb_one(vaddr);
+}
+
+void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
+                   pte_t *ptep, pte_t pteval)
+{
+       if ((mm != current->mm && mm != &init_mm) ||
+           HYPERVISOR_update_va_mapping(addr, pteval, 0) != 0)
+               xen_set_pte(ptep, pteval);
+}
+
+#ifdef CONFIG_X86_PAE
+void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
+{
+       set_64bit((u64 *)ptep, pte_val_ma(pte));
+}
+
+void xen_pte_clear(struct mm_struct *mm, unsigned long addr,pte_t *ptep)
+{
+       ptep->pte_low = 0;
+       smp_wmb();
+       ptep->pte_high = 0;
+}
+
+void xen_pmd_clear(pmd_t *pmdp)
+{
+       xen_set_pmd(pmdp, __pmd(0));
+}
+
+unsigned long long xen_pte_val(pte_t pte)
+{
+       unsigned long long ret = 0;
+
+       if (pte.pte_low) {
+               ret = ((unsigned long long)pte.pte_high << 32) | pte.pte_low;
+               ret = machine_to_phys(XMADDR(ret)).paddr | 1;
+       }
+
+       return ret;
+}
+
+unsigned long long xen_pmd_val(pmd_t pmd)
+{
+       unsigned long long ret = pmd.pmd;
+       if (ret)
+               ret = machine_to_phys(XMADDR(ret)).paddr | 1;
+       return ret;
+}
+
+unsigned long long xen_pgd_val(pgd_t pgd)
+{
+       unsigned long long ret = pgd.pgd;
+       if (ret)
+               ret = machine_to_phys(XMADDR(ret)).paddr | 1;
+       return ret;
+}
+
+pte_t xen_make_pte(unsigned long long pte)
+{
+       if (pte & 1)
+               pte = phys_to_machine(XPADDR(pte)).maddr;
+
+       return (pte_t){ pte, pte >> 32 };
+}
+
+pmd_t xen_make_pmd(unsigned long long pmd)
+{
+       if (pmd & 1)
+               pmd = phys_to_machine(XPADDR(pmd)).maddr;
+
+       return (pmd_t){ pmd };
+}
+
+pgd_t xen_make_pgd(unsigned long long pgd)
+{
+       if (pgd & _PAGE_PRESENT)
+               pgd = phys_to_machine(XPADDR(pgd)).maddr;
+
+       return (pgd_t){ pgd };
+}
+#else  /* !PAE */
+unsigned long xen_pte_val(pte_t pte)
+{
+       unsigned long ret = pte.pte_low;
+
+       if (ret & _PAGE_PRESENT)
+               ret = machine_to_phys(XMADDR(ret)).paddr;
+
+       return ret;
+}
+
+unsigned long xen_pmd_val(pmd_t pmd)
+{
+       /* a BUG here is a lot easier to track down than a NULL eip */
+       BUG();
+       return 0;
+}
+
+unsigned long xen_pgd_val(pgd_t pgd)
+{
+       unsigned long ret = pgd.pgd;
+       if (ret)
+               ret = machine_to_phys(XMADDR(ret)).paddr | 1;
+       return ret;
+}
+
+pte_t xen_make_pte(unsigned long pte)
+{
+       if (pte & _PAGE_PRESENT)
+               pte = phys_to_machine(XPADDR(pte)).maddr;
+
+       return (pte_t){ pte };
+}
+
+pmd_t xen_make_pmd(unsigned long pmd)
+{
+       /* a BUG here is a lot easier to track down than a NULL eip */
+       BUG();
+       return __pmd(0);
+}
+
+pgd_t xen_make_pgd(unsigned long pgd)
+{
+       if (pgd & _PAGE_PRESENT)
+               pgd = phys_to_machine(XPADDR(pgd)).maddr;
+
+       return (pgd_t){ pgd };
+}
+#endif /* CONFIG_X86_PAE */
+
+
+
+static void pgd_walk_set_prot(void *pt, pgprot_t flags)
+{
+       unsigned long pfn = PFN_DOWN(__pa(pt));
+
+       if (HYPERVISOR_update_va_mapping((unsigned long)pt,
+                                        pfn_pte(pfn, flags), 0) < 0)
+               BUG();
+}
+
+static void pgd_walk(pgd_t *pgd_base, pgprot_t flags)
+{
+       pgd_t *pgd = pgd_base;
+       pud_t *pud;
+       pmd_t *pmd;
+       pte_t *pte;
+       int    g, u, m;
+
+       if (xen_feature(XENFEAT_auto_translated_physmap))
+               return;
+
+       for (g = 0; g < USER_PTRS_PER_PGD; g++, pgd++) {
+               if (pgd_none(*pgd))
+                       continue;
+               pud = pud_offset(pgd, 0);
+
+               if (PTRS_PER_PUD > 1) /* not folded */
+                       pgd_walk_set_prot(pud,flags);
+
+               for (u = 0; u < PTRS_PER_PUD; u++, pud++) {
+                       if (pud_none(*pud))
+                               continue;
+                       pmd = pmd_offset(pud, 0);
+
+                       if (PTRS_PER_PMD > 1) /* not folded */
+                               pgd_walk_set_prot(pmd,flags);
+
+                       for (m = 0; m < PTRS_PER_PMD; m++, pmd++) {
+                               if (pmd_none(*pmd))
+                                       continue;
+
+                               /* This can get called before mem_map
+                                  is set up, so we assume nothing is
+                                  highmem at that point. */
+                               if (mem_map == NULL ||
+                                   !PageHighMem(pmd_page(*pmd))) {
+                                       pte = pte_offset_kernel(pmd,0);
+                                       pgd_walk_set_prot(pte,flags);
+                               }
+                       }
+               }
+       }
+
+       if (HYPERVISOR_update_va_mapping((unsigned long)pgd_base,
+                                        pfn_pte(PFN_DOWN(__pa(pgd_base)),
+                                                flags),
+                                        UVMF_TLB_FLUSH) < 0)
+               BUG();
+}
+
+
+/* This is called just after a mm has been duplicated from its parent,
+   but it has not been used yet.  We need to make sure that its
+   pagetable is all read-only, and can be pinned. */
+void xen_pgd_pin(pgd_t *pgd)
+{
+       struct mmuext_op op;
+
+       pgd_walk(pgd, PAGE_KERNEL_RO);
+
+#if defined(CONFIG_X86_PAE)
+       op.cmd = MMUEXT_PIN_L3_TABLE;
+#else
+       op.cmd = MMUEXT_PIN_L2_TABLE;
+#endif
+       op.arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
+       if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
+               BUG();
+}
+
+/* Release a pagetables pages back as normal RW */
+void xen_pgd_unpin(pgd_t *pgd)
+{
+       struct mmuext_op op;
+
+       op.cmd = MMUEXT_UNPIN_TABLE;
+       op.arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd)));
+
+       if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
+               BUG();
+
+       pgd_walk(pgd, PAGE_KERNEL);
+}
+
+
+void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
+{
+       xen_pgd_pin(next->pgd);
+}
+
+void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
+{
+       xen_pgd_pin(mm->pgd);
+}
+
+void xen_exit_mmap(struct mm_struct *mm)
+{
+       struct task_struct *tsk = current;
+
+       task_lock(tsk);
+
+       /*
+        * We aggressively remove defunct pgd from cr3. We execute unmap_vmas()
+        * *much* faster this way, as no tlb flushes means bigger wrpt batches.
+        */
+       if (tsk->active_mm == mm) {
+               tsk->active_mm = &init_mm;
+               atomic_inc(&init_mm.mm_count);
+
+               switch_mm(mm, &init_mm, tsk);
+
+               atomic_dec(&mm->mm_count);
+               BUG_ON(atomic_read(&mm->mm_count) == 0);
+       }
+
+       task_unlock(tsk);
+
+       xen_pgd_unpin(mm->pgd);
+}
===================================================================
--- /dev/null
+++ b/arch/i386/xen/mmu.h
@@ -0,0 +1,47 @@
+#ifndef _XEN_MMU_H
+
+#include <linux/linkage.h>
+#include <asm/page.h>
+
+void set_pte_mfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);
+
+void xen_set_pte(pte_t *ptep, pte_t pteval);
+void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
+                   pte_t *ptep, pte_t pteval);
+void xen_set_pmd(pmd_t *pmdp, pmd_t pmdval);
+
+void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next);
+void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm);
+void xen_exit_mmap(struct mm_struct *mm);
+
+void xen_pgd_pin(pgd_t *pgd);
+void xen_pgd_unpin(pgd_t *pgd);
+
+#ifdef CONFIG_X86_PAE
+unsigned long long xen_pte_val(pte_t);
+unsigned long long xen_pmd_val(pmd_t);
+unsigned long long xen_pgd_val(pgd_t);
+
+pte_t xen_make_pte(unsigned long long);
+pmd_t xen_make_pmd(unsigned long long);
+pgd_t xen_make_pgd(unsigned long long);
+
+void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
+                   pte_t *ptep, pte_t pteval);
+void xen_set_pte_atomic(pte_t *ptep, pte_t pte);
+void xen_set_pud(pud_t *ptr, pud_t val);
+void xen_pte_clear(struct mm_struct *mm, unsigned long addr,pte_t *ptep);
+void xen_pmd_clear(pmd_t *pmdp);
+
+
+#else
+unsigned long xen_pte_val(pte_t);
+unsigned long xen_pmd_val(pmd_t);
+unsigned long xen_pgd_val(pgd_t);
+
+pte_t xen_make_pte(unsigned long);
+pmd_t xen_make_pmd(unsigned long);
+pgd_t xen_make_pgd(unsigned long);
+#endif
+
+#endif /* _XEN_MMU_H */

-- 


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel


 


Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.