# HG changeset patch
# User tdeegan@xxxxxxxxxxxxxxxxxxxxx
# Node ID 0f917d63e9608315e4a925109401e383fc895b2a
# Parent fda70200da01b89d5339342df6c0db372369a16d
Replace shadow pagetable code with shadow2.
---
xen/arch/x86/audit.c | 984 ------
xen/arch/x86/shadow.c | 4150 ----------------------------
xen/arch/x86/shadow32.c | 3782 --------------------------
xen/arch/x86/shadow_guest32.c | 16
xen/arch/x86/shadow_guest32pae.c | 16
xen/arch/x86/shadow_public.c | 2143 --------------
xen/include/asm-x86/shadow_64.h | 587 ----
xen/include/asm-x86/shadow_ops.h | 138
xen/include/asm-x86/shadow_public.h | 61
.hgtags | 10
tools/examples/xmexample.hvm | 4
tools/libxc/xc_domain.c | 13
tools/libxc/xc_hvm_build.c | 13
tools/libxc/xc_linux_build.c | 2
tools/libxc/xc_linux_save.c | 18
tools/libxc/xenctrl.h | 2
tools/misc/xc_shadow.c | 2
tools/python/xen/lowlevel/xc/xc.c | 69
tools/python/xen/xend/XendDomain.py | 24
tools/python/xen/xend/XendDomainInfo.py | 47
tools/python/xen/xend/image.py | 17
tools/python/xen/xm/create.py | 9
xen/arch/x86/Makefile | 16
xen/arch/x86/dom0_ops.c | 2
xen/arch/x86/domain.c | 106
xen/arch/x86/domain_build.c | 13
xen/arch/x86/hvm/hvm.c | 23
xen/arch/x86/hvm/platform.c | 9
xen/arch/x86/hvm/svm/svm.c | 265 -
xen/arch/x86/hvm/svm/vmcb.c | 4
xen/arch/x86/hvm/vlapic.c | 3
xen/arch/x86/hvm/vmx/vmcs.c | 15
xen/arch/x86/hvm/vmx/vmx.c | 228 -
xen/arch/x86/mm.c | 485 +--
xen/arch/x86/setup.c | 2
xen/arch/x86/shadow2-common.c | 3394 +++++++++++++++++++++++
xen/arch/x86/shadow2.c | 4469 +++++++++++++++++++++++++++++++
xen/arch/x86/smpboot.c | 2
xen/arch/x86/traps.c | 32
xen/arch/x86/x86_32/domain_page.c | 33
xen/arch/x86/x86_32/mm.c | 3
xen/arch/x86/x86_64/mm.c | 3
xen/arch/x86/x86_64/traps.c | 14
xen/common/acm_ops.c | 1
xen/common/grant_table.c | 4
xen/common/keyhandler.c | 19
xen/common/memory.c | 11
xen/drivers/char/console.c | 50
xen/include/asm-x86/bitops.h | 18
xen/include/asm-x86/config.h | 22
xen/include/asm-x86/domain.h | 99
xen/include/asm-x86/grant_table.h | 2
xen/include/asm-x86/hvm/hvm.h | 25
xen/include/asm-x86/hvm/support.h | 11
xen/include/asm-x86/hvm/vcpu.h | 6
xen/include/asm-x86/hvm/vmx/vmcs.h | 1
xen/include/asm-x86/hvm/vmx/vmx.h | 49
xen/include/asm-x86/mm.h | 140
xen/include/asm-x86/msr.h | 4
xen/include/asm-x86/page-guest32.h | 7
xen/include/asm-x86/page.h | 37
xen/include/asm-x86/perfc_defn.h | 53
xen/include/asm-x86/processor.h | 1
xen/include/asm-x86/shadow.h | 1791 ------------
xen/include/asm-x86/shadow2-multi.h | 116
xen/include/asm-x86/shadow2-private.h | 612 ++++
xen/include/asm-x86/shadow2-types.h | 705 ++++
xen/include/asm-x86/shadow2.h | 627 ++++
xen/include/asm-x86/x86_32/page-2level.h | 1
xen/include/asm-x86/x86_32/page-3level.h | 3
xen/include/asm-x86/x86_64/page.h | 5
xen/include/public/dom0_ops.h | 16
xen/include/xen/domain_page.h | 13
xen/include/xen/lib.h | 4
xen/include/xen/list.h | 10
xen/include/xen/sched.h | 5
76 files changed, 11147 insertions(+), 14549 deletions(-)
diff -r fda70200da01 -r 0f917d63e960 .hgtags
--- a/.hgtags Wed Aug 16 16:16:32 2006 +0100
+++ b/.hgtags Wed Aug 16 17:02:35 2006 +0100
@@ -15,3 +15,13 @@ c8fdb0caa77b429cf47f9707926e83947778cb48
c8fdb0caa77b429cf47f9707926e83947778cb48 RELEASE-3.0.0
af0573e9e5258db0a9d28aa954dd302ddd2c2d23 3.0.2-rc
d0d3fef37685be264a7f52201f8ef44c030daad3 3.0.2-branched
+6e864d7de9db066f92bea505d256bfe286200fed last-code-review
+a898a6510c5db4e3d1f69d40fcacb540643b0f22 mainline
+bfa6f4a0c594bc0ebd896437d69857b58dab0988 last-code-review
+fc6cbf31bd883bc76ceb97f4b817ac88078d696a latest patch to unstable
+8e55c5c1147589b7a6a1875384d4317aec7ccf84 mainline
+2d2ed4d9b1c14aeee29dfdd77acd6017d31290cd mainline
+0e32095a7b4611d18a82052a9d5b23e474f91af9 mainline
+88e6bd5e2b5439f97e1d50a8724103c619aeaadf mainline
+5233c4b076b9aa073eff63508461b7bfa597737c mainline
+fda70200da01b89d5339342df6c0db372369a16d mainline
diff -r fda70200da01 -r 0f917d63e960 tools/examples/xmexample.hvm
--- a/tools/examples/xmexample.hvm Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/examples/xmexample.hvm Wed Aug 16 17:02:35 2006 +0100
@@ -26,6 +26,10 @@ builder='hvm'
# memory errors. The domain needs enough memory to boot kernel
# and modules. Allocating less than 32MBs is not recommended.
memory = 128
+
+# Shadow pagetable memory for the domain, in MB.
+# Should be at least 2KB per MB of domain memory, plus a few MB per vcpu.
+shadow_memory = 8
# A name for your domain. All domains must have different names.
name = "ExampleHVMDomain"
diff -r fda70200da01 -r 0f917d63e960 tools/libxc/xc_domain.c
--- a/tools/libxc/xc_domain.c Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/libxc/xc_domain.c Wed Aug 16 17:02:35 2006 +0100
@@ -213,21 +213,28 @@ int xc_shadow_control(int xc_handle,
unsigned int sop,
unsigned long *dirty_bitmap,
unsigned long pages,
- xc_shadow_control_stats_t *stats )
+ unsigned long *mb,
+ uint32_t mode,
+ xc_shadow_control_stats_t *stats)
{
int rc;
DECLARE_DOM0_OP;
op.cmd = DOM0_SHADOW_CONTROL;
op.u.shadow_control.domain = (domid_t)domid;
op.u.shadow_control.op = sop;
+ op.u.shadow_control.pages = pages;
+ op.u.shadow_control.mb = mb ? *mb : 0;
+ op.u.shadow_control.mode = mode;
set_xen_guest_handle(op.u.shadow_control.dirty_bitmap, dirty_bitmap);
- op.u.shadow_control.pages = pages;
rc = do_dom0_op(xc_handle, &op);
if ( stats )
memcpy(stats, &op.u.shadow_control.stats,
sizeof(xc_shadow_control_stats_t));
+
+ if ( mb )
+ *mb = op.u.shadow_control.mb;
return (rc == 0) ? op.u.shadow_control.pages : rc;
}
@@ -391,7 +398,7 @@ int xc_domain_memory_populate_physmap(in
if ( err > 0 )
{
- DPRINTF("Failed deallocation for dom %d: %ld pages order %d\n",
+ DPRINTF("Failed allocation for dom %d: %ld pages order %d\n",
domid, nr_extents, extent_order);
errno = EBUSY;
err = -1;
diff -r fda70200da01 -r 0f917d63e960 tools/libxc/xc_hvm_build.c
--- a/tools/libxc/xc_hvm_build.c Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/libxc/xc_hvm_build.c Wed Aug 16 17:02:35 2006 +0100
@@ -395,6 +395,19 @@ static int xc_hvm_build_internal(int xc_
PERROR("Could not get info on domain");
goto error_out;
}
+
+ /* HVM domains must be put into shadow2 mode at the start of day */
+ if ( xc_shadow_control(xc_handle, domid, DOM0_SHADOW2_CONTROL_OP_ENABLE,
+ NULL, 0, NULL,
+ DOM0_SHADOW2_CONTROL_FLAG_ENABLE
+ | DOM0_SHADOW2_CONTROL_FLAG_REFCOUNT
+ | DOM0_SHADOW2_CONTROL_FLAG_TRANSLATE
+ | DOM0_SHADOW2_CONTROL_FLAG_EXTERNAL,
+ NULL) )
+ {
+ PERROR("Could not enable shadow paging for domain.\n");
+ goto error_out;
+ }
memset(ctxt, 0, sizeof(*ctxt));
diff -r fda70200da01 -r 0f917d63e960 tools/libxc/xc_linux_build.c
--- a/tools/libxc/xc_linux_build.c Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/libxc/xc_linux_build.c Wed Aug 16 17:02:35 2006 +0100
@@ -972,7 +972,7 @@ static int setup_guest(int xc_handle,
/* Enable shadow translate mode */
if ( xc_shadow_control(xc_handle, dom,
DOM0_SHADOW_CONTROL_OP_ENABLE_TRANSLATE,
- NULL, 0, NULL) < 0 )
+ NULL, 0, NULL, 0, NULL) < 0 )
{
PERROR("Could not enable translation mode");
goto error_out;
diff -r fda70200da01 -r 0f917d63e960 tools/libxc/xc_linux_save.c
--- a/tools/libxc/xc_linux_save.c Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/libxc/xc_linux_save.c Wed Aug 16 17:02:35 2006 +0100
@@ -338,13 +338,13 @@ static int analysis_phase(int xc_handle,
int i;
xc_shadow_control(xc_handle, domid, DOM0_SHADOW_CONTROL_OP_CLEAN,
- arr, max_pfn, NULL);
+ arr, max_pfn, NULL, 0, NULL);
DPRINTF("#Flush\n");
for ( i = 0; i < 40; i++ ) {
usleep(50000);
now = llgettimeofday();
xc_shadow_control(xc_handle, domid, DOM0_SHADOW_CONTROL_OP_PEEK,
- NULL, 0, &stats);
+ NULL, 0, NULL, 0, &stats);
DPRINTF("now= %lld faults= %" PRId32 " dirty= %" PRId32
" dirty_net= %" PRId32 " dirty_block= %" PRId32"\n",
@@ -727,7 +727,7 @@ int xc_linux_save(int xc_handle, int io_
if (xc_shadow_control(xc_handle, dom,
DOM0_SHADOW_CONTROL_OP_ENABLE_LOGDIRTY,
- NULL, 0, NULL ) < 0) {
+ NULL, 0, NULL, 0, NULL) < 0) {
ERR("Couldn't enable shadow mode");
goto out;
}
@@ -879,7 +879,7 @@ int xc_linux_save(int xc_handle, int io_
but this is fast enough for the moment. */
if (!last_iter && xc_shadow_control(
xc_handle, dom, DOM0_SHADOW_CONTROL_OP_PEEK,
- to_skip, max_pfn, NULL) != max_pfn) {
+ to_skip, max_pfn, NULL, 0, NULL) != max_pfn) {
ERR("Error peeking shadow bitmap");
goto out;
}
@@ -1084,8 +1084,9 @@ int xc_linux_save(int xc_handle, int io_
(unsigned long)ctxt.user_regs.edx);
}
- if (xc_shadow_control(xc_handle, dom, DOM0_SHADOW_CONTROL_OP_CLEAN,
- to_send, max_pfn, &stats ) != max_pfn) {
+ if (xc_shadow_control(xc_handle, dom,
+ DOM0_SHADOW_CONTROL_OP_CLEAN, to_send,
+ max_pfn, NULL, 0, &stats) != max_pfn) {
ERR("Error flushing shadow PT");
goto out;
}
@@ -1174,8 +1175,9 @@ int xc_linux_save(int xc_handle, int io_
out:
if (live) {
- if(xc_shadow_control(xc_handle, dom, DOM0_SHADOW_CONTROL_OP_OFF,
- NULL, 0, NULL ) < 0) {
+ if(xc_shadow_control(xc_handle, dom,
+ DOM0_SHADOW_CONTROL_OP_OFF,
+ NULL, 0, NULL, 0, NULL) < 0) {
DPRINTF("Warning - couldn't disable shadow mode");
}
}
diff -r fda70200da01 -r 0f917d63e960 tools/libxc/xenctrl.h
--- a/tools/libxc/xenctrl.h Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/libxc/xenctrl.h Wed Aug 16 17:02:35 2006 +0100
@@ -323,6 +323,8 @@ int xc_shadow_control(int xc_handle,
unsigned int sop,
unsigned long *dirty_bitmap,
unsigned long pages,
+ unsigned long *mb,
+ uint32_t mode,
xc_shadow_control_stats_t *stats);
int xc_bvtsched_global_set(int xc_handle,
diff -r fda70200da01 -r 0f917d63e960 tools/misc/xc_shadow.c
--- a/tools/misc/xc_shadow.c Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/misc/xc_shadow.c Wed Aug 16 17:02:35 2006 +0100
@@ -60,6 +60,8 @@ int main(int argc, char *argv[])
mode,
NULL,
0,
+ NULL,
+ 0,
NULL) < 0 )
{
fprintf(stderr, "Error reseting performance counters: %d (%s)\n",
diff -r fda70200da01 -r 0f917d63e960 tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/python/xen/lowlevel/xc/xc.c Wed Aug 16 17:02:35 2006 +0100
@@ -669,6 +669,59 @@ static PyObject *pyxc_sedf_domain_get(Xc
"weight", weight);
}
+static PyObject *pyxc_shadow_control(PyObject *self,
+ PyObject *args,
+ PyObject *kwds)
+{
+ XcObject *xc = (XcObject *)self;
+
+ uint32_t dom;
+ int op=0;
+
+ static char *kwd_list[] = { "dom", "op", NULL };
+
+ if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|i", kwd_list,
+ &dom, &op) )
+ return NULL;
+
+ if ( xc_shadow_control(xc->xc_handle, dom, op, NULL, 0, NULL, 0, NULL)
+ < 0 )
+ return PyErr_SetFromErrno(xc_error);
+
+ Py_INCREF(zero);
+ return zero;
+}
+
+static PyObject *pyxc_shadow_mem_control(PyObject *self,
+ PyObject *args,
+ PyObject *kwds)
+{
+ XcObject *xc = (XcObject *)self;
+ int op;
+ uint32_t dom;
+ int mbarg = -1;
+ unsigned long mb;
+
+ static char *kwd_list[] = { "dom", "mb", NULL };
+
+ if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|i", kwd_list,
+ &dom, &mbarg) )
+ return NULL;
+
+ if ( mbarg < 0 )
+ op = DOM0_SHADOW2_CONTROL_OP_GET_ALLOCATION;
+ else
+ {
+ mb = mbarg;
+ op = DOM0_SHADOW2_CONTROL_OP_SET_ALLOCATION;
+ }
+ if ( xc_shadow_control(xc->xc_handle, dom, op, NULL, 0, &mb, 0, NULL) < 0 )
+ return PyErr_SetFromErrno(xc_error);
+
+ mbarg = mb;
+ return Py_BuildValue("i", mbarg);
+}
+
static PyObject *pyxc_sched_credit_domain_set(XcObject *self,
PyObject *args,
PyObject *kwds)
@@ -1118,6 +1171,22 @@ static PyMethodDef pyxc_methods[] = {
"Get information about the Xen host\n"
"Returns [dict]: information about Xen"
" [None]: on failure.\n" },
+
+ { "shadow_control",
+ (PyCFunction)pyxc_shadow_control,
+ METH_VARARGS | METH_KEYWORDS, "\n"
+ "Set parameter for shadow pagetable interface\n"
+ " dom [int]: Identifier of domain.\n"
+ " op [int, 0]: operation\n\n"
+ "Returns: [int] 0 on success; -1 on error.\n" },
+
+ { "shadow_mem_control",
+ (PyCFunction)pyxc_shadow_mem_control,
+ METH_VARARGS | METH_KEYWORDS, "\n"
+ "Set or read shadow pagetable memory use\n"
+ " dom [int]: Identifier of domain.\n"
+ " mb [int, -1]: MB of shadow memory this domain should have.\n\n"
+ "Returns: [int] MB of shadow memory in use by this domain.\n" },
{ "domain_setmaxmem",
(PyCFunction)pyxc_domain_setmaxmem,
diff -r fda70200da01 -r 0f917d63e960 tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/python/xen/xend/XendDomain.py Wed Aug 16 17:02:35 2006 +0100
@@ -532,6 +532,30 @@ class XendDomain:
except Exception, ex:
raise XendError(str(ex))
+ def domain_shadow_control(self, domid, op):
+ """Shadow page control."""
+ dominfo = self.domain_lookup(domid)
+ try:
+ return xc.shadow_control(dominfo.getDomid(), op)
+ except Exception, ex:
+ raise XendError(str(ex))
+
+ def domain_shadow_mem_get(self, domid):
+ """Get shadow pagetable memory allocation."""
+ dominfo = self.domain_lookup(domid)
+ try:
+ return xc.shadow_mem_control(dominfo.getDomid())
+ except Exception, ex:
+ raise XendError(str(ex))
+
+ def domain_shadow_mem_set(self, domid, mb):
+ """Set shadow pagetable memory allocation."""
+ dominfo = self.domain_lookup(domid)
+ try:
+ return xc.shadow_mem_control(dominfo.getDomid(), mb=mb)
+ except Exception, ex:
+ raise XendError(str(ex))
+
def domain_sched_credit_get(self, domid):
"""Get credit scheduler parameters for a domain.
"""
diff -r fda70200da01 -r 0f917d63e960 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/python/xen/xend/XendDomainInfo.py Wed Aug 16 17:02:35 2006 +0100
@@ -30,6 +30,7 @@ import time
import time
import threading
import os
+import math
import xen.lowlevel.xc
from xen.util import asserts
@@ -126,16 +127,17 @@ VM_CONFIG_PARAMS = [
# don't come out of xc in the same form as they are specified in the config
# file, so those are handled separately.
ROUNDTRIPPING_CONFIG_ENTRIES = [
- ('uuid', str),
- ('vcpus', int),
- ('vcpu_avail', int),
- ('cpu_weight', float),
- ('memory', int),
- ('maxmem', int),
- ('bootloader', str),
+ ('uuid', str),
+ ('vcpus', int),
+ ('vcpu_avail', int),
+ ('cpu_weight', float),
+ ('memory', int),
+ ('shadow_memory', int),
+ ('maxmem', int),
+ ('bootloader', str),
('bootloader_args', str),
- ('features', str),
- ('localtime', int),
+ ('features', str),
+ ('localtime', int),
]
ROUNDTRIPPING_CONFIG_ENTRIES += VM_CONFIG_PARAMS
@@ -146,12 +148,13 @@ ROUNDTRIPPING_CONFIG_ENTRIES += VM_CONFI
# entries written to the store that cannot be reconfigured on-the-fly.
#
VM_STORE_ENTRIES = [
- ('uuid', str),
- ('vcpus', int),
- ('vcpu_avail', int),
- ('memory', int),
- ('maxmem', int),
- ('start_time', float),
+ ('uuid', str),
+ ('vcpus', int),
+ ('vcpu_avail', int),
+ ('memory', int),
+ ('shadow_memory', int),
+ ('maxmem', int),
+ ('start_time', float),
]
VM_STORE_ENTRIES += VM_CONFIG_PARAMS
@@ -572,6 +575,7 @@ class XendDomainInfo:
defaultInfo('vcpu_avail', lambda: (1 << self.info['vcpus']) - 1)
defaultInfo('memory', lambda: 0)
+ defaultInfo('shadow_memory', lambda: 0)
defaultInfo('maxmem', lambda: 0)
defaultInfo('bootloader', lambda: None)
defaultInfo('bootloader_args', lambda: None)
@@ -1280,7 +1284,18 @@ class XendDomainInfo:
xc.domain_setmaxmem(self.domid, self.info['maxmem'] * 1024)
m = self.image.getDomainMemory(self.info['memory'] * 1024)
- balloon.free(m)
+
+ # get the domain's shadow memory requirement
+ sm = int(math.ceil(self.image.getDomainShadowMemory(m) / 1024.0))
+ if self.info['shadow_memory'] > sm:
+ sm = self.info['shadow_memory']
+
+ # Make sure there's enough RAM available for the domain
+ balloon.free(m + sm * 1024)
+
+ # Set up the shadow memory
+ sm = xc.shadow_mem_control(self.domid, mb=sm)
+ self.info['shadow_memory'] = sm
init_reservation = self.info['memory'] * 1024
if os.uname()[4] in ('ia64', 'ppc64'):
diff -r fda70200da01 -r 0f917d63e960 tools/python/xen/xend/image.py
--- a/tools/python/xen/xend/image.py Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/python/xen/xend/image.py Wed Aug 16 17:02:35 2006 +0100
@@ -152,6 +152,12 @@ class ImageHandler:
if 'hvm' in xc.xeninfo()['xen_caps']:
mem_kb += 4*1024;
return mem_kb
+
+ def getDomainShadowMemory(self, mem_kb):
+ """@return The minimum shadow memory required, in KiB, for a domain
+ with mem_kb KiB of RAM."""
+ # PV domains don't need any shadow memory
+ return 0
def buildDomain(self):
"""Build the domain. Define in subclass."""
@@ -364,6 +370,17 @@ class HVMImageHandler(ImageHandler):
extra_pages = int( math.ceil( extra_mb*1024 / page_kb ))
return mem_kb + extra_pages * page_kb
+ def getDomainShadowMemory(self, mem_kb):
+ """@return The minimum shadow memory required, in KiB, for a domain
+ with mem_kb KiB of RAM."""
+ if os.uname()[4] in ('ia64', 'ppc64'):
+ # Explicit shadow memory is not a concept
+ return 0
+ else:
+ # 1MB per vcpu plus 4Kib/Mib of RAM. This is higher than
+ # the minimum that Xen would allocate if no value were given.
+ return 1024 * self.vm.getVCpuCount() + mem_kb / 256
+
def register_shutdown_watch(self):
""" add xen store watch on control/shutdown """
self.shutdownWatch = xswatch(self.vm.dompath + "/control/shutdown", \
diff -r fda70200da01 -r 0f917d63e960 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py Wed Aug 16 16:16:32 2006 +0100
+++ b/tools/python/xen/xm/create.py Wed Aug 16 17:02:35 2006 +0100
@@ -157,6 +157,10 @@ gopts.var('maxmem', val='MEMORY',
gopts.var('maxmem', val='MEMORY',
fn=set_int, default=None,
use="Maximum domain memory in MB.")
+
+gopts.var('shadow_memory', val='MEMORY',
+ fn=set_int, default=0,
+ use="Domain shadow memory in MB.")
gopts.var('cpu', val='CPU',
fn=set_int, default=None,
@@ -666,8 +670,9 @@ def make_config(vals):
if v:
config.append([n, v])
- map(add_conf, ['name', 'memory', 'maxmem', 'restart', 'on_poweroff',
- 'on_reboot', 'on_crash', 'vcpus', 'features'])
+ map(add_conf, ['name', 'memory', 'maxmem', 'shadow_memory',
+ 'restart', 'on_poweroff', 'on_reboot', 'on_crash',
+ 'vcpus', 'features'])
if vals.uuid is not None:
config.append(['uuid', vals.uuid])
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/Makefile
--- a/xen/arch/x86/Makefile Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/Makefile Wed Aug 16 17:02:35 2006 +0100
@@ -8,7 +8,6 @@ subdir-$(x86_64) += x86_64
subdir-$(x86_64) += x86_64
obj-y += apic.o
-obj-y += audit.o
obj-y += bitops.o
obj-y += compat.o
obj-y += delay.o
@@ -41,12 +40,21 @@ obj-y += x86_emulate.o
obj-y += x86_emulate.o
ifneq ($(pae),n)
-obj-$(x86_32) += shadow.o shadow_public.o shadow_guest32.o shadow_guest32pae.o
+obj-$(x86_32) += shadow2-common.o shadow2_g2_on_s3.o shadow2_g3_on_s3.o
else
-obj-$(x86_32) += shadow32.o
+obj-$(x86_32) += shadow2-common.o shadow2_g2_on_s2.o
endif
-obj-$(x86_64) += shadow.o shadow_public.o shadow_guest32.o shadow_guest32pae.o
+obj-$(x86_64) += shadow2-common.o shadow2_g4_on_s4.o shadow2_g3_on_s3.o \
+ shadow2_g2_on_s3.o
+
+guest_levels = $(subst g,,$(filter g%,$(subst ., ,$(subst _, ,$(subst
shadow2_,,$(1))))))
+shadow_levels = $(subst s,,$(filter s%,$(subst ., ,$(subst _, ,$(subst
shadow2_,,$(1))))))
+shadow2_defns = -DGUEST_PAGING_LEVELS=$(call guest_levels,$(1)) \
+ -DSHADOW_PAGING_LEVELS=$(call shadow_levels,$(1))
+
+shadow2_%.o: shadow2.c $(HDRS) Makefile
+ $(CC) $(CFLAGS) $(call shadow2_defns,$(@F)) -c $< -o $@
obj-$(crash_debug) += gdbstub.o
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/dom0_ops.c
--- a/xen/arch/x86/dom0_ops.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/dom0_ops.c Wed Aug 16 17:02:35 2006 +0100
@@ -89,7 +89,7 @@ long arch_do_dom0_op(struct dom0_op *op,
d = find_domain_by_id(op->u.shadow_control.domain);
if ( d != NULL )
{
- ret = shadow_mode_control(d, &op->u.shadow_control);
+ ret = shadow2_control_op(d, &op->u.shadow_control, u_dom0_op);
put_domain(d);
copy_to_guest(u_dom0_op, op, 1);
}
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/domain.c
--- a/xen/arch/x86/domain.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/domain.c Wed Aug 16 17:02:35 2006 +0100
@@ -134,13 +134,6 @@ struct vcpu *alloc_vcpu_struct(struct do
v->arch.perdomain_ptes =
d->arch.mm_perdomain_pt + (vcpu_id << GDT_LDT_VCPU_SHIFT);
- v->arch.guest_vtable = __linear_l2_table;
- v->arch.shadow_vtable = __shadow_linear_l2_table;
-#if defined(__x86_64__)
- v->arch.guest_vl3table = __linear_l3_table;
- v->arch.guest_vl4table = __linear_l4_table;
-#endif
-
pae_l3_cache_init(&v->arch.pae_l3_cache);
return v;
@@ -155,9 +148,7 @@ int arch_domain_create(struct domain *d)
{
l1_pgentry_t gdt_l1e;
int vcpuid, pdpt_order;
-#ifdef __x86_64__
int i;
-#endif
pdpt_order = get_order_from_bytes(PDPT_L1_ENTRIES * sizeof(l1_pgentry_t));
d->arch.mm_perdomain_pt = alloc_xenheap_pages(pdpt_order);
@@ -202,8 +193,12 @@ int arch_domain_create(struct domain *d)
#endif /* __x86_64__ */
- shadow_lock_init(d);
- INIT_LIST_HEAD(&d->arch.free_shadow_frames);
+ shadow2_lock_init(d);
+ for ( i = 0; i <= SHADOW2_MAX_ORDER; i++ )
+ INIT_LIST_HEAD(&d->arch.shadow2_freelists[i]);
+ INIT_LIST_HEAD(&d->arch.shadow2_p2m_freelist);
+ INIT_LIST_HEAD(&d->arch.shadow2_p2m_inuse);
+ INIT_LIST_HEAD(&d->arch.shadow2_toplevel_shadows);
if ( !is_idle_domain(d) )
{
@@ -234,6 +229,8 @@ int arch_domain_create(struct domain *d)
void arch_domain_destroy(struct domain *d)
{
+ shadow2_final_teardown(d);
+
free_xenheap_pages(
d->arch.mm_perdomain_pt,
get_order_from_bytes(PDPT_L1_ENTRIES * sizeof(l1_pgentry_t)));
@@ -328,31 +325,35 @@ int arch_set_info_guest(
if ( !hvm_initialize_guest_resources(v) )
return -EINVAL;
}
- else if ( shadow_mode_refcounts(d) )
- {
- if ( !get_page(mfn_to_page(cr3_pfn), d) )
+ else
+ {
+ if ( !get_page_and_type(mfn_to_page(cr3_pfn), d,
+ PGT_base_page_table) )
{
destroy_gdt(v);
return -EINVAL;
}
- }
- else
- {
- if ( !get_page_and_type(mfn_to_page(cr3_pfn), d,
- PGT_base_page_table) )
- {
- destroy_gdt(v);
- return -EINVAL;
- }
- }
-
- update_pagetables(v);
+ }
+
+ /* Shadow2: make sure the domain has enough shadow memory to
+ * boot another vcpu */
+ if ( shadow2_mode_enabled(d)
+ && d->arch.shadow2_total_pages < shadow2_min_acceptable_pages(d) )
+ {
+ destroy_gdt(v);
+ return -ENOMEM;
+ }
if ( v->vcpu_id == 0 )
update_domain_wallclock_time(d);
/* Don't redo final setup */
set_bit(_VCPUF_initialised, &v->vcpu_flags);
+
+ if ( shadow2_mode_enabled(d) )
+ shadow2_update_paging_modes(v);
+
+ update_cr3(v);
return 0;
}
@@ -669,7 +670,6 @@ static void __context_switch(void)
loaddebug(&n->arch.guest_context, 6);
loaddebug(&n->arch.guest_context, 7);
}
-
n->arch.ctxt_switch_to(n);
}
@@ -927,29 +927,34 @@ void domain_relinquish_resources(struct
/* Drop the in-use references to page-table bases. */
for_each_vcpu ( d, v )
{
- if ( (pfn = pagetable_get_pfn(v->arch.guest_table)) != 0 )
- {
- if ( !shadow_mode_refcounts(d) )
- put_page_type(mfn_to_page(pfn));
- put_page(mfn_to_page(pfn));
-
+ /* Drop ref to guest_table (from new_guest_cr3(), svm/vmx cr3 handling,
+ * or sh2_update_paging_modes()) */
+ pfn = pagetable_get_pfn(v->arch.guest_table);
+ if ( pfn != 0 )
+ {
+ if ( shadow2_mode_refcounts(d) )
+ put_page(mfn_to_page(pfn));
+ else
+ put_page_and_type(mfn_to_page(pfn));
v->arch.guest_table = pagetable_null();
}
- if ( (pfn = pagetable_get_pfn(v->arch.guest_table_user)) != 0 )
- {
- if ( !shadow_mode_refcounts(d) )
- put_page_type(mfn_to_page(pfn));
- put_page(mfn_to_page(pfn));
-
+#ifdef __x86_64__
+ /* Drop ref to guest_table_user (from MMUEXT_NEW_USER_BASEPTR) */
+ pfn = pagetable_get_pfn(v->arch.guest_table_user);
+ if ( pfn != 0 )
+ {
+ put_page_and_type(mfn_to_page(pfn));
v->arch.guest_table_user = pagetable_null();
}
+#endif
}
if ( d->vcpu[0] && hvm_guest(d->vcpu[0]) )
hvm_relinquish_guest_resources(d);
- shadow_mode_disable(d);
+ /* Tear down shadow mode stuff. */
+ shadow2_teardown(d);
/*
* Relinquish GDT mappings. No need for explicit unmapping of the LDT as
@@ -964,26 +969,23 @@ void domain_relinquish_resources(struct
/* Free page used by xen oprofile buffer */
free_xenoprof_pages(d);
-
}
void arch_dump_domain_info(struct domain *d)
{
- if ( shadow_mode_enabled(d) )
- {
- printk(" shadow mode: ");
- if ( shadow_mode_refcounts(d) )
+ if ( shadow2_mode_enabled(d) )
+ {
+ printk(" shadow2 mode: ");
+ if ( d->arch.shadow2_mode & SHM2_enable )
+ printk("enabled ");
+ if ( shadow2_mode_refcounts(d) )
printk("refcounts ");
- if ( shadow_mode_write_all(d) )
- printk("write_all ");
- if ( shadow_mode_log_dirty(d) )
+ if ( shadow2_mode_log_dirty(d) )
printk("log_dirty ");
- if ( shadow_mode_translate(d) )
+ if ( shadow2_mode_translate(d) )
printk("translate ");
- if ( shadow_mode_external(d) )
+ if ( shadow2_mode_external(d) )
printk("external ");
- if ( shadow_mode_wr_pt_pte(d) )
- printk("wr_pt_pte ");
printk("\n");
}
}
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/domain_build.c
--- a/xen/arch/x86/domain_build.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/domain_build.c Wed Aug 16 17:02:35 2006 +0100
@@ -683,8 +683,11 @@ int construct_dom0(struct domain *d,
for ( i = 1; i < opt_dom0_max_vcpus; i++ )
(void)alloc_vcpu(d, i, i);
- /* Set up monitor table */
- update_pagetables(v);
+ /* Set up CR3 value for write_ptbase */
+ if ( shadow2_mode_enabled(v->domain) )
+ shadow2_update_paging_modes(v);
+ else
+ update_cr3(v);
/* Install the new page tables. */
local_irq_disable();
@@ -796,10 +799,8 @@ int construct_dom0(struct domain *d,
new_thread(v, dsi.v_kernentry, vstack_end, vstartinfo_start);
if ( opt_dom0_shadow )
- {
- shadow_mode_enable(d, SHM_enable);
- update_pagetables(v);
- }
+ if ( shadow2_test_enable(d) == 0 )
+ shadow2_update_paging_modes(v);
if ( supervisor_mode_kernel )
{
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/hvm.c Wed Aug 16 17:02:35 2006 +0100
@@ -30,6 +30,7 @@
#include <xen/hypercall.h>
#include <xen/guest_access.h>
#include <xen/event.h>
+#include <xen/shadow.h>
#include <asm/current.h>
#include <asm/e820.h>
#include <asm/io.h>
@@ -42,10 +43,6 @@
#include <asm/spinlock.h>
#include <asm/hvm/hvm.h>
#include <asm/hvm/support.h>
-#include <asm/shadow.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
#include <public/sched.h>
#include <public/hvm/ioreq.h>
#include <public/version.h>
@@ -61,7 +58,7 @@ static void hvm_zap_mmio_range(
static void hvm_zap_mmio_range(
struct domain *d, unsigned long pfn, unsigned long nr_pfn)
{
- unsigned long i, val = INVALID_MFN;
+ unsigned long i;
ASSERT(d == current->domain);
@@ -70,7 +67,8 @@ static void hvm_zap_mmio_range(
if ( pfn + i >= 0xfffff )
break;
- __copy_to_user(&phys_to_machine_mapping[pfn + i], &val, sizeof (val));
+ if ( VALID_MFN(gmfn_to_mfn(d, pfn + i)) )
+ guest_remove_page(d, pfn + i);
}
}
@@ -262,11 +260,13 @@ void hvm_setup_platform(struct domain* d
if ( !hvm_guest(v) || (v->vcpu_id != 0) )
return;
+#if 0 /* SHADOW2 does not have this */
if ( shadow_direct_map_init(d) == 0 )
{
printk("Can not allocate shadow direct map for HVM domain.\n");
domain_crash_synchronous();
}
+#endif
hvm_zap_iommu_pages(d);
@@ -380,6 +380,8 @@ void hvm_hlt(unsigned long rflags)
*/
int hvm_copy(void *buf, unsigned long vaddr, int size, int dir)
{
+ struct vcpu *v = current;
+ unsigned long gfn;
unsigned long mfn;
char *addr;
int count;
@@ -389,10 +391,9 @@ int hvm_copy(void *buf, unsigned long va
if (count > size)
count = size;
- if (hvm_paging_enabled(current))
- mfn = gva_to_mfn(vaddr);
- else
- mfn = get_mfn_from_gpfn(vaddr >> PAGE_SHIFT);
+ gfn = shadow2_gva_to_gfn(v, vaddr);
+ mfn = mfn_x(sh2_vcpu_gfn_to_mfn(v, gfn));
+
if (mfn == INVALID_MFN)
return 0;
@@ -545,7 +546,7 @@ void hvm_do_hypercall(struct cpu_user_re
return;
}
- if ( current->domain->arch.ops->guest_paging_levels == PAGING_L4 )
+ if ( current->arch.shadow2->guest_levels == 4 )
{
pregs->rax = hvm_hypercall64_table[pregs->rax](pregs->rdi,
pregs->rsi,
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/platform.c
--- a/xen/arch/x86/hvm/platform.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/platform.c Wed Aug 16 17:02:35 2006 +0100
@@ -21,7 +21,7 @@
#include <xen/config.h>
#include <xen/types.h>
#include <xen/mm.h>
-#include <asm/shadow.h>
+#include <xen/shadow.h>
#include <xen/domain_page.h>
#include <asm/page.h>
#include <xen/event.h>
@@ -35,9 +35,6 @@
#include <xen/lib.h>
#include <xen/sched.h>
#include <asm/current.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
#define DECODE_success 1
#define DECODE_failure 0
@@ -724,7 +721,7 @@ void send_pio_req(struct cpu_user_regs *
if (pvalid) {
if (hvm_paging_enabled(current))
- p->u.pdata = (void *) gva_to_gpa(value);
+ p->u.data = shadow2_gva_to_gpa(current, value);
else
p->u.pdata = (void *) value; /* guest VA == guest PA */
} else
@@ -774,7 +771,7 @@ void send_mmio_req(
if (pvalid) {
if (hvm_paging_enabled(v))
- p->u.pdata = (void *) gva_to_gpa(value);
+ p->u.data = shadow2_gva_to_gpa(v, value);
else
p->u.pdata = (void *) value; /* guest VA == guest PA */
} else
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/svm/svm.c
--- a/xen/arch/x86/hvm/svm/svm.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/svm/svm.c Wed Aug 16 17:02:35 2006 +0100
@@ -26,9 +26,10 @@
#include <xen/irq.h>
#include <xen/softirq.h>
#include <xen/hypercall.h>
+#include <xen/domain_page.h>
#include <asm/current.h>
#include <asm/io.h>
-#include <asm/shadow.h>
+#include <asm/shadow2.h>
#include <asm/regs.h>
#include <asm/cpufeature.h>
#include <asm/processor.h>
@@ -43,10 +44,6 @@
#include <asm/hvm/svm/emulate.h>
#include <asm/hvm/svm/vmmcall.h>
#include <asm/hvm/svm/intr.h>
-#include <asm/shadow.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
#include <public/sched.h>
#define SVM_EXTRA_DEBUG
@@ -414,7 +411,7 @@ static int svm_realmode(struct vcpu *v)
return (eflags & X86_EFLAGS_VM) || !(cr0 & X86_CR0_PE);
}
-static int svm_instruction_length(struct vcpu *v)
+int svm_guest_x86_mode(struct vcpu *v)
{
struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
unsigned long cr0 = vmcb->cr0, eflags = vmcb->rflags, mode;
@@ -423,10 +420,20 @@ static int svm_instruction_length(struct
mode = vmcb->cs.attributes.fields.l ? 8 : 4;
else
mode = (eflags & X86_EFLAGS_VM) || !(cr0 & X86_CR0_PE) ? 2 : 4;
- return svm_instrlen(guest_cpu_user_regs(), mode);
-}
-
-static unsigned long svm_get_ctrl_reg(struct vcpu *v, unsigned int num)
+ return mode;
+}
+
+int svm_instruction_length(struct vcpu *v)
+{
+ return svm_instrlen(guest_cpu_user_regs(), svm_guest_x86_mode(v));
+}
+
+void svm_update_host_cr3(struct vcpu *v)
+{
+ /* SVM doesn't have a HOST_CR3 equivalent to update. */
+}
+
+unsigned long svm_get_ctrl_reg(struct vcpu *v, unsigned int num)
{
switch ( num )
{
@@ -436,6 +443,8 @@ static unsigned long svm_get_ctrl_reg(st
return v->arch.hvm_svm.cpu_cr2;
case 3:
return v->arch.hvm_svm.cpu_cr3;
+ case 4:
+ return v->arch.hvm_svm.cpu_shadow_cr4;
default:
BUG();
}
@@ -524,8 +533,6 @@ static void svm_init_hypercall_page(stru
/* Don't support HYPERVISOR_iret at the moment */
*(u16 *)(hypercall_page + (__HYPERVISOR_iret * 32)) = 0x0b0f; /* ud2 */
}
-
-
int svm_dbg_on = 0;
@@ -647,6 +654,11 @@ static void svm_load_cpu_guest_regs(
svm_load_cpu_user_regs(v, regs);
}
+int svm_long_mode_enabled(struct vcpu *v)
+{
+ return SVM_LONG_GUEST(v);
+}
+
static void arch_svm_do_launch(struct vcpu *v)
@@ -726,7 +738,6 @@ static void svm_final_setup_guest(struct
static void svm_final_setup_guest(struct vcpu *v)
{
struct domain *d = v->domain;
- struct vcpu *vc;
v->arch.schedule_tail = arch_svm_do_launch;
v->arch.ctxt_switch_from = svm_ctxt_switch_from;
@@ -735,9 +746,12 @@ static void svm_final_setup_guest(struct
if ( v != d->vcpu[0] )
return;
- /* Initialize monitor page table */
- for_each_vcpu( d, vc )
- vc->arch.monitor_table = pagetable_null();
+ if ( !shadow2_mode_external(d) )
+ {
+ DPRINTK("Can't init HVM for dom %u vcpu %u: "
+ "not in shadow2 external mode\n", d->domain_id, v->vcpu_id);
+ domain_crash(d);
+ }
/*
* Required to do this once per domain
@@ -745,13 +759,6 @@ static void svm_final_setup_guest(struct
*/
memset(&d->shared_info->evtchn_mask[0], 0xff,
sizeof(d->shared_info->evtchn_mask));
-
- /*
- * Put the domain in shadow mode even though we're going to be using
- * the shared 1:1 page table initially. It shouldn't hurt
- */
- shadow_mode_enable(d, SHM_enable|SHM_refcounts|
- SHM_translate|SHM_external|SHM_wr_pt_pte);
}
@@ -809,9 +816,13 @@ int start_svm(void)
hvm_funcs.realmode = svm_realmode;
hvm_funcs.paging_enabled = svm_paging_enabled;
+ hvm_funcs.long_mode_enabled = svm_long_mode_enabled;
+ hvm_funcs.guest_x86_mode = svm_guest_x86_mode;
hvm_funcs.instruction_length = svm_instruction_length;
hvm_funcs.get_guest_ctrl_reg = svm_get_ctrl_reg;
+ hvm_funcs.update_host_cr3 = svm_update_host_cr3;
+
hvm_funcs.stts = svm_stts;
hvm_funcs.set_tsc_offset = svm_set_tsc_offset;
@@ -834,7 +845,6 @@ static void svm_relinquish_guest_resourc
continue;
destroy_vmcb(&v->arch.hvm_svm);
- free_monitor_pagetable(v);
kill_timer(&v->arch.hvm_vcpu.hlt_timer);
if ( hvm_apic_support(v->domain) && (VLAPIC(v) != NULL) )
{
@@ -851,8 +861,6 @@ static void svm_relinquish_guest_resourc
if ( d->arch.hvm_domain.buffered_io_va )
unmap_domain_page_global((void *)d->arch.hvm_domain.buffered_io_va);
-
- shadow_direct_map_clean(d);
}
@@ -894,7 +902,6 @@ static int svm_do_page_fault(unsigned lo
{
struct vcpu *v = current;
unsigned long eip;
- unsigned long gpa; /* FIXME: PAE */
int result;
struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
@@ -907,43 +914,7 @@ static int svm_do_page_fault(unsigned lo
va, eip, (unsigned long)regs->error_code);
//#endif
- if ( !svm_paging_enabled(v) )
- {
- if ( shadow_direct_map_fault(va, regs) )
- return 1;
-
- handle_mmio(va, va);
- return 1;
- }
-
-
- gpa = gva_to_gpa(va);
-
- /* Use 1:1 page table to identify MMIO address space */
- if (mmio_space(gpa))
- {
- /* No support for APIC */
- if (!hvm_apic_support(v->domain) && gpa >= 0xFEC00000)
- {
- int inst_len;
- inst_len = svm_instruction_length(v);
- if (inst_len == -1)
- {
- printf("%s: INST_LEN - Unable to decode properly\n", __func__);
- domain_crash_synchronous();
- }
-
- __update_guest_eip(vmcb, inst_len);
-
- return 1;
- }
-
- handle_mmio(va, gpa);
-
- return 1;
- }
-
- result = shadow_fault(va, regs);
+ result = shadow2_fault(va, regs);
if( result ) {
/* Let's make sure that the Guest TLB is flushed */
@@ -1035,19 +1006,12 @@ static void svm_vmexit_do_cpuid(struct v
clear_bit(X86_FEATURE_APIC, &edx);
}
-#if CONFIG_PAGING_LEVELS < 3
- clear_bit(X86_FEATURE_PAE, &edx);
- clear_bit(X86_FEATURE_PSE, &edx);
+#if CONFIG_PAGING_LEVELS >= 3
+ if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
+#endif
+ clear_bit(X86_FEATURE_PAE, &edx);
clear_bit(X86_FEATURE_PSE36, &edx);
-#else
- if ( v->domain->arch.ops->guest_paging_levels == PAGING_L2 )
- {
- if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
- clear_bit(X86_FEATURE_PAE, &edx);
- clear_bit(X86_FEATURE_PSE, &edx);
- clear_bit(X86_FEATURE_PSE36, &edx);
- }
-#endif
+
/* Clear out reserved bits. */
ecx &= ~SVM_VCPU_CPUID_L1_ECX_RESERVED;
edx &= ~SVM_VCPU_CPUID_L1_EDX_RESERVED;
@@ -1097,23 +1061,12 @@ static void svm_vmexit_do_cpuid(struct v
clear_bit(X86_FEATURE_SYSCALL & 31, &edx);
#endif
-#if CONFIG_PAGING_LEVELS < 3
- clear_bit(X86_FEATURE_NX & 31, &edx);
- clear_bit(X86_FEATURE_PAE, &edx);
- clear_bit(X86_FEATURE_PSE, &edx);
+
+#if CONFIG_PAGING_LEVELS >= 3
+ if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
+#endif
+ clear_bit(X86_FEATURE_PAE, &edx);
clear_bit(X86_FEATURE_PSE36, &edx);
-#else
- if ( v->domain->arch.ops->guest_paging_levels == PAGING_L2 )
- {
- if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
- {
- clear_bit(X86_FEATURE_NX & 31, &edx);
- clear_bit(X86_FEATURE_PAE, &edx);
- }
- clear_bit(X86_FEATURE_PSE, &edx);
- clear_bit(X86_FEATURE_PSE36, &edx);
- }
-#endif
/* Make SVM feature invisible to the guest. */
clear_bit(X86_FEATURE_SVME & 31, &ecx);
@@ -1555,6 +1508,7 @@ static int svm_set_cr0(unsigned long val
unsigned long mfn;
int paging_enabled;
struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
+ unsigned long old_base_mfn;
ASSERT(vmcb);
@@ -1600,54 +1554,21 @@ static int svm_set_cr0(unsigned long val
set_bit(SVM_CPU_STATE_LMA_ENABLED,
&v->arch.hvm_svm.cpu_state);
vmcb->efer |= (EFER_LMA | EFER_LME);
- if (!shadow_set_guest_paging_levels(v->domain, PAGING_L4) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous(); /* need to take a clean path */
- }
- }
- else
+ }
#endif /* __x86_64__ */
- {
-#if CONFIG_PAGING_LEVELS >= 3
- /* seems it's a 32-bit or 32-bit PAE guest */
- if ( test_bit(SVM_CPU_STATE_PAE_ENABLED,
- &v->arch.hvm_svm.cpu_state) )
- {
- /* The guest enables PAE first and then it enables PG, it is
- * really a PAE guest */
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous();
- }
- }
- else
- {
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L2) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous(); /* need to take a clean path */
- }
- }
-#endif
- }
/* Now arch.guest_table points to machine physical. */
+ old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
v->arch.guest_table = pagetable_from_pfn(mfn);
- update_pagetables(v);
+ if ( old_base_mfn )
+ put_page(mfn_to_page(old_base_mfn));
+ shadow2_update_paging_modes(v);
HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
(unsigned long) (mfn << PAGE_SHIFT));
+ vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
- vmcb->cr3 = pagetable_get_paddr(v->arch.shadow_table);
-
- /* arch->shadow_table should hold the next CR3 for shadow */
- HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx, mfn = %lx\n",
- v->arch.hvm_svm.cpu_cr3, mfn);
-
- return 1;
}
if ( !((value & X86_CR0_PE) && (value & X86_CR0_PG)) && paging_enabled )
@@ -1667,17 +1588,16 @@ static int svm_set_cr0(unsigned long val
svm_inject_exception(v, TRAP_gp_fault, 1, 0);
return 0;
}
-
- clear_all_shadow_status( v->domain );
+ shadow2_update_paging_modes(v);
+ vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
- vmcb->cr3 = pagetable_get_paddr(v->domain->arch.phys_table);
}
else if ( (value & (X86_CR0_PE | X86_CR0_PG)) == X86_CR0_PE )
{
/* we should take care of this kind of situation */
- clear_all_shadow_status(v->domain);
+ shadow2_update_paging_modes(v);
+ vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
- vmcb->cr3 = pagetable_get_paddr(v->domain->arch.phys_table);
}
return 1;
@@ -1786,7 +1706,7 @@ static int mov_to_cr(int gpreg, int cr,
mfn = get_mfn_from_gpfn(value >> PAGE_SHIFT);
if (mfn != pagetable_get_pfn(v->arch.guest_table))
__hvm_bug(regs);
- shadow_sync_all(v->domain);
+ shadow2_update_cr3(v);
}
else
{
@@ -1812,14 +1732,10 @@ static int mov_to_cr(int gpreg, int cr,
/*
* arch.shadow_table should now hold the next CR3 for shadow
*/
-#if CONFIG_PAGING_LEVELS >= 3
- if ( v->domain->arch.ops->guest_paging_levels == PAGING_L3 )
- shadow_sync_all(v->domain);
-#endif
v->arch.hvm_svm.cpu_cr3 = value;
- update_pagetables(v);
+ update_cr3(v);
+ vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx", value);
- vmcb->cr3 = pagetable_get_paddr(v->arch.shadow_table);
}
break;
}
@@ -1838,12 +1754,6 @@ static int mov_to_cr(int gpreg, int cr,
/* The guest is a 32-bit PAE guest. */
#if CONFIG_PAGING_LEVELS >= 3
unsigned long mfn, old_base_mfn;
-
- if( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous(); /* need to take a clean path */
- }
if ( !VALID_MFN(mfn = get_mfn_from_gpfn(
v->arch.hvm_svm.cpu_cr3 >> PAGE_SHIFT)) ||
@@ -1853,21 +1763,20 @@ static int mov_to_cr(int gpreg, int cr,
domain_crash_synchronous(); /* need to take a clean path */
}
- old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
- if ( old_base_mfn )
- put_page(mfn_to_page(old_base_mfn));
-
/*
* Now arch.guest_table points to machine physical.
*/
+ old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
v->arch.guest_table = pagetable_from_pfn(mfn);
- update_pagetables(v);
+ if ( old_base_mfn )
+ put_page(mfn_to_page(old_base_mfn));
+ shadow2_update_paging_modes(v);
HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
(unsigned long) (mfn << PAGE_SHIFT));
- vmcb->cr3 = pagetable_get_paddr(v->arch.shadow_table);
+ vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
/*
* arch->shadow_table should hold the next CR3 for shadow
@@ -1876,33 +1785,6 @@ static int mov_to_cr(int gpreg, int cr,
HVM_DBG_LOG(DBG_LEVEL_VMMU,
"Update CR3 value = %lx, mfn = %lx",
v->arch.hvm_svm.cpu_cr3, mfn);
-#endif
- }
- else
- {
- /* The guest is a 64 bit or 32-bit PAE guest. */
-#if CONFIG_PAGING_LEVELS >= 3
- if ( (v->domain->arch.ops != NULL) &&
- v->domain->arch.ops->guest_paging_levels == PAGING_L2)
- {
- /* Seems the guest first enables PAE without enabling PG,
- * it must enable PG after that, and it is a 32-bit PAE
- * guest */
-
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L3))
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous();
- }
- }
- else
- {
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L4))
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous();
- }
- }
#endif
}
}
@@ -1926,7 +1808,7 @@ static int mov_to_cr(int gpreg, int cr,
if ((old_cr ^ value) & (X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE))
{
set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
- shadow_sync_all(v->domain);
+ shadow2_update_paging_modes(v);
}
break;
}
@@ -2267,7 +2149,7 @@ void svm_handle_invlpg(const short invlp
/* Overkill, we may not this */
set_bit(ARCH_SVM_VMCB_ASSIGN_ASID, &v->arch.hvm_svm.flags);
- shadow_invlpg(v, g_vaddr);
+ shadow2_invlpg(v, g_vaddr);
}
@@ -2638,7 +2520,7 @@ void walk_shadow_and_guest_pt(unsigned l
struct vmcb_struct *vmcb = v->arch.hvm_svm.vmcb;
unsigned long gpa;
- gpa = gva_to_gpa( gva );
+ gpa = shadow2_gva_to_gpa(current, gva);
printk( "gva = %lx, gpa=%lx, gCR3=%x\n", gva, gpa, (u32)vmcb->cr3 );
if( !svm_paging_enabled(v) || mmio_space(gpa) )
return;
@@ -2662,8 +2544,12 @@ void walk_shadow_and_guest_pt(unsigned l
__copy_from_user(&gpte, &linear_pg_table[ l1_linear_offset(gva) ],
sizeof(gpte) );
printk( "G-PTE = %x, flags=%x\n", gpte.l1, l1e_get_flags(gpte) );
- __copy_from_user( &spte, &phys_to_machine_mapping[ l1e_get_pfn( gpte ) ],
+
+ BUG(); // need to think about this, and convert usage of
+ // phys_to_machine_mapping to use pagetable format...
+ __copy_from_user( &spte, &phys_to_machine_mapping[ l1e_get_pfn( gpte ) ],
sizeof(spte) );
+
printk( "S-PTE = %x, flags=%x\n", spte.l1, l1e_get_flags(spte));
}
#endif /* SVM_WALK_GUEST_PAGES */
@@ -2704,7 +2590,8 @@ asmlinkage void svm_vmexit_handler(struc
if (svm_dbg_on && exit_reason == VMEXIT_EXCEPTION_PF)
{
- if (svm_paging_enabled(v) && !mmio_space(gva_to_gpa(vmcb->exitinfo2)))
+ if (svm_paging_enabled(v) &&
+ !mmio_space(shadow2_gva_to_gpa(current, vmcb->exitinfo2)))
{
printk("I%08ld,ExC=%s(%d),IP=%x:%llx,I1=%llx,I2=%llx,INT=%llx, "
"gpa=%llx\n", intercepts_counter,
@@ -2713,7 +2600,7 @@ asmlinkage void svm_vmexit_handler(struc
(unsigned long long) vmcb->exitinfo1,
(unsigned long long) vmcb->exitinfo2,
(unsigned long long) vmcb->exitintinfo.bytes,
- (unsigned long long) gva_to_gpa( vmcb->exitinfo2 ) );
+ (unsigned long long) shadow2_gva_to_gpa(current, vmcb->exitinfo2));
}
else
{
@@ -2757,7 +2644,7 @@ asmlinkage void svm_vmexit_handler(struc
&& ( ( vmcb->exitinfo2 == vmcb->rip )
|| vmcb->exitintinfo.bytes) )
{
- if (svm_paging_enabled(v) && !mmio_space(gva_to_gpa(vmcb->exitinfo2)))
+ if (svm_paging_enabled(v) && !mmio_space(gva_to_gpa(vmcb->exitinfo2)))
walk_shadow_and_guest_pt( vmcb->exitinfo2 );
}
#endif
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/svm/vmcb.c
--- a/xen/arch/x86/hvm/svm/vmcb.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/svm/vmcb.c Wed Aug 16 17:02:35 2006 +0100
@@ -380,8 +380,8 @@ void svm_do_launch(struct vcpu *v)
printk("%s: phys_table = %lx\n", __func__, pt);
}
- /* At launch we always use the phys_table */
- vmcb->cr3 = pagetable_get_paddr(v->domain->arch.phys_table);
+ /* Set cr3 from hw_cr3 even when guest-visible paging is not enabled */
+ vmcb->cr3 = v->arch.hvm_vcpu.hw_cr3;
if (svm_dbg_on)
{
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/vlapic.c
--- a/xen/arch/x86/hvm/vlapic.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/vlapic.c Wed Aug 16 17:02:35 2006 +0100
@@ -21,7 +21,8 @@
#include <xen/types.h>
#include <xen/mm.h>
#include <xen/xmalloc.h>
-#include <asm/shadow.h>
+#include <xen/shadow.h>
+#include <xen/domain_page.h>
#include <asm/page.h>
#include <xen/event.h>
#include <xen/trace.h>
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/vmx/vmcs.c
--- a/xen/arch/x86/hvm/vmx/vmcs.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/vmx/vmcs.c Wed Aug 16 17:02:35 2006 +0100
@@ -34,12 +34,8 @@
#include <asm/flushtlb.h>
#include <xen/event.h>
#include <xen/kernel.h>
-#include <asm/shadow.h>
#include <xen/keyhandler.h>
-
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
+#include <asm/shadow2.h>
static int vmcs_size;
static int vmcs_order;
@@ -238,7 +234,7 @@ static void vmx_set_host_env(struct vcpu
static void vmx_do_launch(struct vcpu *v)
{
-/* Update CR3, GDT, LDT, TR */
+/* Update CR3, CR0, CR4, GDT, LDT, TR */
unsigned int error = 0;
unsigned long cr0, cr4;
@@ -276,8 +272,11 @@ static void vmx_do_launch(struct vcpu *v
error |= __vmwrite(GUEST_TR_BASE, 0);
error |= __vmwrite(GUEST_TR_LIMIT, 0xff);
- __vmwrite(GUEST_CR3, pagetable_get_paddr(v->domain->arch.phys_table));
- __vmwrite(HOST_CR3, pagetable_get_paddr(v->arch.monitor_table));
+ shadow2_update_paging_modes(v);
+ printk("%s(): GUEST_CR3<=%08lx, HOST_CR3<=%08lx\n",
+ __func__, v->arch.hvm_vcpu.hw_cr3, v->arch.cr3);
+ __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
+ __vmwrite(HOST_CR3, v->arch.cr3);
v->arch.schedule_tail = arch_vmx_do_resume;
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/hvm/vmx/vmx.c Wed Aug 16 17:02:35 2006 +0100
@@ -26,9 +26,9 @@
#include <xen/softirq.h>
#include <xen/domain_page.h>
#include <xen/hypercall.h>
+#include <xen/perfc.h>
#include <asm/current.h>
#include <asm/io.h>
-#include <asm/shadow.h>
#include <asm/regs.h>
#include <asm/cpufeature.h>
#include <asm/processor.h>
@@ -40,10 +40,7 @@
#include <asm/hvm/vmx/vmx.h>
#include <asm/hvm/vmx/vmcs.h>
#include <asm/hvm/vmx/cpu.h>
-#include <asm/shadow.h>
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/shadow_64.h>
-#endif
+#include <asm/shadow2.h>
#include <public/sched.h>
#include <public/hvm/ioreq.h>
#include <asm/hvm/vpic.h>
@@ -69,11 +66,16 @@ static int vmx_initialize_guest_resource
if ( v->vcpu_id != 0 )
return 1;
+ if ( !shadow2_mode_external(d) )
+ {
+ DPRINTK("Can't init HVM for dom %u vcpu %u: "
+ "not in shadow2 external mode\n",
+ d->domain_id, v->vcpu_id);
+ domain_crash(d);
+ }
+
for_each_vcpu ( d, vc )
{
- /* Initialize monitor page table */
- vc->arch.monitor_table = pagetable_null();
-
memset(&vc->arch.hvm_vmx, 0, sizeof(struct arch_vmx_struct));
if ( (rc = vmx_create_vmcs(vc)) != 0 )
@@ -107,6 +109,7 @@ static int vmx_initialize_guest_resource
vc->arch.hvm_vmx.io_bitmap_a = io_bitmap_a;
vc->arch.hvm_vmx.io_bitmap_b = io_bitmap_b;
+
}
/*
@@ -116,11 +119,6 @@ static int vmx_initialize_guest_resource
memset(&d->shared_info->evtchn_mask[0], 0xff,
sizeof(d->shared_info->evtchn_mask));
- /* Put the domain in shadow mode even though we're going to be using
- * the shared 1:1 page table initially. It shouldn't hurt */
- shadow_mode_enable(
- d, SHM_enable|SHM_refcounts|SHM_translate|SHM_external|SHM_wr_pt_pte);
-
return 1;
}
@@ -133,7 +131,6 @@ static void vmx_relinquish_guest_resourc
vmx_destroy_vmcs(v);
if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) )
continue;
- free_monitor_pagetable(v);
kill_timer(&v->arch.hvm_vcpu.hlt_timer);
if ( hvm_apic_support(v->domain) && (VLAPIC(v) != NULL) )
{
@@ -153,8 +150,6 @@ static void vmx_relinquish_guest_resourc
if ( d->arch.hvm_domain.buffered_io_va )
unmap_domain_page_global((void *)d->arch.hvm_domain.buffered_io_va);
-
- shadow_direct_map_clean(d);
}
#ifdef __x86_64__
@@ -595,14 +590,6 @@ static void vmx_load_cpu_guest_regs(stru
vmx_vmcs_exit(v);
}
-static int vmx_realmode(struct vcpu *v)
-{
- unsigned long rflags;
-
- __vmread(GUEST_RFLAGS, &rflags);
- return rflags & X86_EFLAGS_VM;
-}
-
static int vmx_instruction_length(struct vcpu *v)
{
unsigned long inst_len;
@@ -622,6 +609,8 @@ static unsigned long vmx_get_ctrl_reg(st
return v->arch.hvm_vmx.cpu_cr2;
case 3:
return v->arch.hvm_vmx.cpu_cr3;
+ case 4:
+ return v->arch.hvm_vmx.cpu_shadow_cr4;
default:
BUG();
}
@@ -753,8 +742,12 @@ static void vmx_setup_hvm_funcs(void)
hvm_funcs.realmode = vmx_realmode;
hvm_funcs.paging_enabled = vmx_paging_enabled;
+ hvm_funcs.long_mode_enabled = vmx_long_mode_enabled;
+ hvm_funcs.guest_x86_mode = vmx_guest_x86_mode;
hvm_funcs.instruction_length = vmx_instruction_length;
hvm_funcs.get_guest_ctrl_reg = vmx_get_ctrl_reg;
+
+ hvm_funcs.update_host_cr3 = vmx_update_host_cr3;
hvm_funcs.stts = vmx_stts;
hvm_funcs.set_tsc_offset = vmx_set_tsc_offset;
@@ -855,53 +848,25 @@ static void inline __update_guest_eip(un
__vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0);
}
-
static int vmx_do_page_fault(unsigned long va, struct cpu_user_regs *regs)
{
- unsigned long gpa; /* FIXME: PAE */
int result;
#if 0 /* keep for debugging */
{
- unsigned long eip;
-
+ unsigned long eip, cs;
+
+ __vmread(GUEST_CS_BASE, &cs);
__vmread(GUEST_RIP, &eip);
HVM_DBG_LOG(DBG_LEVEL_VMMU,
- "vmx_do_page_fault = 0x%lx, eip = %lx, error_code = %lx",
- va, eip, (unsigned long)regs->error_code);
+ "vmx_do_page_fault = 0x%lx, cs_base=%lx, "
+ "eip = %lx, error_code = %lx\n",
+ va, cs, eip, (unsigned long)regs->error_code);
}
#endif
- if ( !vmx_paging_enabled(current) )
- {
- /* construct 1-to-1 direct mapping */
- if ( shadow_direct_map_fault(va, regs) )
- return 1;
-
- handle_mmio(va, va);
- TRACE_VMEXIT (2,2);
- return 1;
- }
- gpa = gva_to_gpa(va);
-
- /* Use 1:1 page table to identify MMIO address space */
- if ( mmio_space(gpa) ){
- struct vcpu *v = current;
- /* No support for APIC */
- if (!hvm_apic_support(v->domain) && gpa >= 0xFEC00000) {
- u32 inst_len;
- __vmread(VM_EXIT_INSTRUCTION_LEN, &(inst_len));
- __update_guest_eip(inst_len);
- return 1;
- }
- TRACE_VMEXIT (2,2);
- /* in the case of MMIO, we are more interested in gpa than in va */
- TRACE_VMEXIT (4,gpa);
- handle_mmio(va, gpa);
- return 1;
- }
-
- result = shadow_fault(va, regs);
+ result = shadow2_fault(va, regs);
+
TRACE_VMEXIT (2,result);
#if 0
if ( !result )
@@ -972,23 +937,11 @@ static void vmx_vmexit_do_cpuid(struct c
clear_bit(X86_FEATURE_APIC, &edx);
}
-#if CONFIG_PAGING_LEVELS < 3
- edx &= ~(bitmaskof(X86_FEATURE_PAE) |
- bitmaskof(X86_FEATURE_PSE) |
- bitmaskof(X86_FEATURE_PSE36));
-#else
- if ( v->domain->arch.ops->guest_paging_levels == PAGING_L2 )
- {
- if ( v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
- clear_bit(X86_FEATURE_PSE36, &edx);
- else
- {
- clear_bit(X86_FEATURE_PAE, &edx);
- clear_bit(X86_FEATURE_PSE, &edx);
- clear_bit(X86_FEATURE_PSE36, &edx);
- }
- }
+#if CONFIG_PAGING_LEVELS >= 3
+ if ( !v->domain->arch.hvm_domain.params[HVM_PARAM_PAE_ENABLED] )
#endif
+ clear_bit(X86_FEATURE_PAE, &edx);
+ clear_bit(X86_FEATURE_PSE36, &edx);
ebx &= NUM_THREADS_RESET_MASK;
@@ -1086,7 +1039,7 @@ static void vmx_vmexit_do_invlpg(unsigne
* We do the safest things first, then try to update the shadow
* copying from guest
*/
- shadow_invlpg(v, va);
+ shadow2_invlpg(v, va);
}
@@ -1307,11 +1260,8 @@ vmx_world_restore(struct vcpu *v, struct
error |= __vmwrite(CR0_READ_SHADOW, c->cr0);
- if (!vmx_paging_enabled(v)) {
- HVM_DBG_LOG(DBG_LEVEL_VMMU, "switching to vmxassist. use phys table");
- __vmwrite(GUEST_CR3, pagetable_get_paddr(v->domain->arch.phys_table));
+ if (!vmx_paging_enabled(v))
goto skip_cr3;
- }
if (c->cr3 == v->arch.hvm_vmx.cpu_cr3) {
/*
@@ -1325,7 +1275,6 @@ vmx_world_restore(struct vcpu *v, struct
domain_crash_synchronous();
return 0;
}
- shadow_sync_all(v->domain);
} else {
/*
* If different, make a shadow. Check if the PDBR is valid
@@ -1348,12 +1297,16 @@ vmx_world_restore(struct vcpu *v, struct
* arch.shadow_table should now hold the next CR3 for shadow
*/
v->arch.hvm_vmx.cpu_cr3 = c->cr3;
- update_pagetables(v);
+ }
+
+ skip_cr3:
+
+ shadow2_update_paging_modes(v);
+ if (!vmx_paging_enabled(v))
+ HVM_DBG_LOG(DBG_LEVEL_VMMU, "switching to vmxassist. use phys table");
+ else
HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %x", c->cr3);
- __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
- }
-
- skip_cr3:
+ __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
error |= __vmread(CR4_READ_SHADOW, &old_cr4);
error |= __vmwrite(GUEST_CR4, (c->cr4 | VMX_CR4_HOST_MASK));
@@ -1485,6 +1438,7 @@ static int vmx_set_cr0(unsigned long val
int paging_enabled;
unsigned long vm_entry_value;
unsigned long old_cr0;
+ unsigned long old_base_mfn;
/*
* CR0: We don't want to lose PE and PG.
@@ -1514,7 +1468,8 @@ static int vmx_set_cr0(unsigned long val
v->arch.hvm_vmx.cpu_cr3 >> PAGE_SHIFT)) ||
!get_page(mfn_to_page(mfn), v->domain) )
{
- printk("Invalid CR3 value = %lx", v->arch.hvm_vmx.cpu_cr3);
+ printk("Invalid CR3 value = %lx (mfn=%lx)\n",
+ v->arch.hvm_vmx.cpu_cr3, mfn);
domain_crash_synchronous(); /* need to take a clean path */
}
@@ -1539,51 +1494,22 @@ static int vmx_set_cr0(unsigned long val
__vmread(VM_ENTRY_CONTROLS, &vm_entry_value);
vm_entry_value |= VM_ENTRY_CONTROLS_IA32E_MODE;
__vmwrite(VM_ENTRY_CONTROLS, vm_entry_value);
-
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L4) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous(); /* need to take a clean path */
- }
- }
- else
-#endif /* __x86_64__ */
- {
-#if CONFIG_PAGING_LEVELS >= 3
- /* seems it's a 32-bit or 32-bit PAE guest */
-
- if ( test_bit(VMX_CPU_STATE_PAE_ENABLED,
- &v->arch.hvm_vmx.cpu_state) )
- {
- /* The guest enables PAE first and then it enables PG, it is
- * really a PAE guest */
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous();
- }
- }
- else
- {
- if ( !shadow_set_guest_paging_levels(v->domain, PAGING_L2) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous(); /* need to take a clean path */
- }
- }
+ }
#endif
- }
/*
* Now arch.guest_table points to machine physical.
*/
+ old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
v->arch.guest_table = pagetable_from_pfn(mfn);
- update_pagetables(v);
+ if (old_base_mfn)
+ put_page(mfn_to_page(old_base_mfn));
+ shadow2_update_paging_modes(v);
HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
(unsigned long) (mfn << PAGE_SHIFT));
- __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
+ __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
/*
* arch->shadow_table should hold the next CR3 for shadow
*/
@@ -1625,7 +1551,6 @@ static int vmx_set_cr0(unsigned long val
}
}
- clear_all_shadow_status(v->domain);
if ( vmx_assist(v, VMX_ASSIST_INVOKE) ) {
set_bit(VMX_CPU_STATE_ASSIST_ENABLED, &v->arch.hvm_vmx.cpu_state);
__vmread(GUEST_RIP, &eip);
@@ -1651,9 +1576,8 @@ static int vmx_set_cr0(unsigned long val
}
else if ( (value & (X86_CR0_PE | X86_CR0_PG)) == X86_CR0_PE )
{
- /* we should take care of this kind of situation */
- clear_all_shadow_status(v->domain);
- __vmwrite(GUEST_CR3, pagetable_get_paddr(v->domain->arch.phys_table));
+ __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
+ shadow2_update_paging_modes(v);
}
return 1;
@@ -1738,7 +1662,7 @@ static int mov_to_cr(int gp, int cr, str
mfn = get_mfn_from_gpfn(value >> PAGE_SHIFT);
if (mfn != pagetable_get_pfn(v->arch.guest_table))
__hvm_bug(regs);
- shadow_sync_all(v->domain);
+ shadow2_update_cr3(v);
} else {
/*
* If different, make a shadow. Check if the PDBR is valid
@@ -1759,16 +1683,11 @@ static int mov_to_cr(int gp, int cr, str
/*
* arch.shadow_table should now hold the next CR3 for shadow
*/
-#if CONFIG_PAGING_LEVELS >= 3
- if ( v->domain->arch.ops->guest_paging_levels == PAGING_L3 )
- shadow_sync_all(v->domain);
-#endif
-
v->arch.hvm_vmx.cpu_cr3 = value;
- update_pagetables(v);
+ update_cr3(v);
HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx",
value);
- __vmwrite(GUEST_CR3, pagetable_get_paddr(v->arch.shadow_table));
+ __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
}
break;
}
@@ -1785,12 +1704,6 @@ static int mov_to_cr(int gp, int cr, str
/* The guest is a 32-bit PAE guest. */
#if CONFIG_PAGING_LEVELS >= 3
unsigned long mfn, old_base_mfn;
-
- if( !shadow_set_guest_paging_levels(v->domain, PAGING_L3) )
- {
- printk("Unsupported guest paging levels\n");
- domain_crash_synchronous(); /* need to take a clean path */
- }
if ( !VALID_MFN(mfn = get_mfn_from_gpfn(
v->arch.hvm_vmx.cpu_cr3 >> PAGE_SHIFT)) ||
@@ -1800,21 +1713,20 @@ static int mov_to_cr(int gp, int cr, str
domain_crash_synchronous(); /* need to take a clean path */
}
- old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
- if ( old_base_mfn )
- put_page(mfn_to_page(old_base_mfn));
/*
* Now arch.guest_table points to machine physical.
*/
+ old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
v->arch.guest_table = pagetable_from_pfn(mfn);
- update_pagetables(v);
+ if ( old_base_mfn )
+ put_page(mfn_to_page(old_base_mfn));
HVM_DBG_LOG(DBG_LEVEL_VMMU, "New arch.guest_table = %lx",
(unsigned long) (mfn << PAGE_SHIFT));
- __vmwrite(GUEST_CR3,
pagetable_get_paddr(v->arch.shadow_table));
+ __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr3);
/*
* arch->shadow_table should hold the next CR3 for shadow
@@ -1822,27 +1734,6 @@ static int mov_to_cr(int gp, int cr, str
HVM_DBG_LOG(DBG_LEVEL_VMMU, "Update CR3 value = %lx, mfn =
%lx",
v->arch.hvm_vmx.cpu_cr3, mfn);
-#endif
- }
- else
- {
- /* The guest is a 64 bit or 32-bit PAE guest. */
-#if CONFIG_PAGING_LEVELS >= 3
- if ( (v->domain->arch.ops != NULL) &&
- v->domain->arch.ops->guest_paging_levels == PAGING_L2)
- {
- /* Seems the guest first enables PAE without enabling PG,
- * it must enable PG after that, and it is a 32-bit PAE
- * guest */
-
- if ( !shadow_set_guest_paging_levels(v->domain,
- PAGING_L3) )
- {
- printk("Unsupported guest paging levels\n");
- /* need to take a clean path */
- domain_crash_synchronous();
- }
- }
#endif
}
}
@@ -1864,8 +1755,7 @@ static int mov_to_cr(int gp, int cr, str
* all TLB entries except global entries.
*/
if ( (old_cr ^ value) & (X86_CR4_PSE | X86_CR4_PGE | X86_CR4_PAE) )
- shadow_sync_all(v->domain);
-
+ shadow2_update_paging_modes(v);
break;
}
default:
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/mm.c
--- a/xen/arch/x86/mm.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/mm.c Wed Aug 16 17:02:35 2006 +0100
@@ -137,7 +137,7 @@ static void free_l1_table(struct page_in
static int mod_l2_entry(l2_pgentry_t *, l2_pgentry_t, unsigned long,
unsigned long type);
-static int mod_l1_entry(l1_pgentry_t *, l1_pgentry_t);
+static int mod_l1_entry(l1_pgentry_t *, l1_pgentry_t, unsigned long gl1mfn);
/* Used to defer flushing of memory structures. */
struct percpu_mm_info {
@@ -274,9 +274,9 @@ void share_xen_page_with_privileged_gues
#else
/*
* In debug builds we shadow a selection of <4GB PDPTs to exercise code paths.
- * We cannot safely shadow the idle page table, nor shadow-mode page tables
+ * We cannot safely shadow the idle page table, nor shadow (v1) page tables
* (detected by lack of an owning domain). As required for correctness, we
- * always shadow PDPTs aboive 4GB.
+ * always shadow PDPTs above 4GB.
*/
#define l3tab_needs_shadow(mfn) \
(((((mfn) << PAGE_SHIFT) != __pa(idle_pg_table)) && \
@@ -297,17 +297,21 @@ static int __init cache_pae_fixmap_addre
}
__initcall(cache_pae_fixmap_address);
-static void __write_ptbase(unsigned long mfn)
+static DEFINE_PER_CPU(u32, make_cr3_timestamp);
+
+void make_cr3(struct vcpu *v, unsigned long mfn)
+/* Takes the MFN of a PAE l3 table, copies the contents to below 4GB if
+ * necessary, and sets v->arch.cr3 to the value to load in CR3. */
{
l3_pgentry_t *highmem_l3tab, *lowmem_l3tab;
- struct pae_l3_cache *cache = ¤t->arch.pae_l3_cache;
+ struct pae_l3_cache *cache = &v->arch.pae_l3_cache;
unsigned int cpu = smp_processor_id();
- /* Fast path 1: does this mfn need a shadow at all? */
+ /* Fast path: does this mfn need a shadow at all? */
if ( !l3tab_needs_shadow(mfn) )
{
- write_cr3(mfn << PAGE_SHIFT);
- /* Cache is no longer in use or valid (/after/ write to %cr3). */
+ v->arch.cr3 = mfn << PAGE_SHIFT;
+ /* Cache is no longer in use or valid */
cache->high_mfn = 0;
return;
}
@@ -315,13 +319,6 @@ static void __write_ptbase(unsigned long
/* Caching logic is not interrupt safe. */
ASSERT(!in_irq());
- /* Fast path 2: is this mfn already cached? */
- if ( cache->high_mfn == mfn )
- {
- write_cr3(__pa(cache->table[cache->inuse_idx]));
- return;
- }
-
/* Protects against pae_flush_pgd(). */
spin_lock(&cache->lock);
@@ -330,29 +327,33 @@ static void __write_ptbase(unsigned long
/* Map the guest L3 table and copy to the chosen low-memory cache. */
*(fix_pae_highmem_pl1e - cpu) = l1e_from_pfn(mfn, __PAGE_HYPERVISOR);
+ /* First check the previous high mapping can't be in the TLB.
+ * (i.e. have we loaded CR3 since we last did this?) */
+ if ( unlikely(this_cpu(make_cr3_timestamp) == this_cpu(tlbflush_time)) )
+ local_flush_tlb_one(fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu));
highmem_l3tab = (l3_pgentry_t *)fix_to_virt(FIX_PAE_HIGHMEM_0 + cpu);
lowmem_l3tab = cache->table[cache->inuse_idx];
memcpy(lowmem_l3tab, highmem_l3tab, sizeof(cache->table[0]));
*(fix_pae_highmem_pl1e - cpu) = l1e_empty();
-
- /* Install the low-memory L3 table in CR3. */
- write_cr3(__pa(lowmem_l3tab));
+ this_cpu(make_cr3_timestamp) = this_cpu(tlbflush_time);
+
+ v->arch.cr3 = __pa(lowmem_l3tab);
spin_unlock(&cache->lock);
}
#else /* !CONFIG_X86_PAE */
-static void __write_ptbase(unsigned long mfn)
-{
- write_cr3(mfn << PAGE_SHIFT);
+void make_cr3(struct vcpu *v, unsigned long mfn)
+{
+ v->arch.cr3 = mfn << PAGE_SHIFT;
}
#endif /* !CONFIG_X86_PAE */
void write_ptbase(struct vcpu *v)
{
- __write_ptbase(pagetable_get_pfn(v->arch.monitor_table));
+ write_cr3(v->arch.cr3);
}
void invalidate_shadow_ldt(struct vcpu *v)
@@ -423,8 +424,6 @@ int map_ldt_shadow_page(unsigned int off
BUG_ON(unlikely(in_irq()));
- shadow_sync_va(v, gva);
-
TOGGLE_MODE();
__copy_from_user(&l1e, &linear_pg_table[l1_linear_offset(gva)],
sizeof(l1e));
@@ -440,12 +439,12 @@ int map_ldt_shadow_page(unsigned int off
res = get_page_and_type(mfn_to_page(mfn), d, PGT_ldt_page);
- if ( !res && unlikely(shadow_mode_refcounts(d)) )
- {
- shadow_lock(d);
- shadow_remove_all_write_access(d, gmfn, mfn);
+ if ( !res && unlikely(shadow2_mode_refcounts(d)) )
+ {
+ shadow2_lock(d);
+ shadow2_remove_write_access(d->vcpu[0], _mfn(mfn), 0, 0);
res = get_page_and_type(mfn_to_page(mfn), d, PGT_ldt_page);
- shadow_unlock(d);
+ shadow2_unlock(d);
}
if ( unlikely(!res) )
@@ -513,7 +512,7 @@ get_linear_pagetable(
struct page_info *page;
unsigned long pfn;
- ASSERT( !shadow_mode_refcounts(d) );
+ ASSERT( !shadow2_mode_refcounts(d) );
if ( (root_get_flags(re) & _PAGE_RW) )
{
@@ -576,7 +575,8 @@ get_page_from_l1e(
if ( !iomem_access_permitted(d, mfn, mfn) )
{
- MEM_LOG("Non-privileged attempt to map I/O space %08lx", mfn);
+ MEM_LOG("Non-privileged (%u) attempt to map I/O space %08lx",
+ d->domain_id, mfn);
return 0;
}
@@ -587,9 +587,14 @@ get_page_from_l1e(
d = dom_io;
}
- okay = ((l1e_get_flags(l1e) & _PAGE_RW) ?
- get_page_and_type(page, d, PGT_writable_page) :
- get_page(page, d));
+ /* Foreign mappings into guests in shadow2 external mode don't
+ * contribute to writeable mapping refcounts. (This allows the
+ * qemu-dm helper process in dom0 to map the domain's memory without
+ * messing up the count of "real" writable mappings.) */
+ okay = (((l1e_get_flags(l1e) & _PAGE_RW) &&
+ !(unlikely(shadow2_mode_external(d) && (d != current->domain))))
+ ? get_page_and_type(page, d, PGT_writable_page)
+ : get_page(page, d));
if ( !okay )
{
MEM_LOG("Error getting mfn %lx (pfn %lx) from L1 entry %" PRIpte
@@ -609,8 +614,6 @@ get_page_from_l2e(
struct domain *d, unsigned long vaddr)
{
int rc;
-
- ASSERT(!shadow_mode_refcounts(d));
if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) )
return 1;
@@ -641,8 +644,6 @@ get_page_from_l3e(
{
int rc;
- ASSERT(!shadow_mode_refcounts(d));
-
if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
return 1;
@@ -668,8 +669,6 @@ get_page_from_l4e(
struct domain *d, unsigned long vaddr)
{
int rc;
-
- ASSERT( !shadow_mode_refcounts(d) );
if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) )
return 1;
@@ -727,7 +726,10 @@ void put_page_from_l1e(l1_pgentry_t l1e,
domain_crash(d);
}
- if ( l1e_get_flags(l1e) & _PAGE_RW )
+ /* Remember we didn't take a type-count of foreign writable mappings
+ * to shadow2 external domains */
+ if ( (l1e_get_flags(l1e) & _PAGE_RW) &&
+ !(unlikely((e != d) && shadow2_mode_external(e))) )
{
put_page_and_type(page);
}
@@ -784,7 +786,7 @@ static int alloc_l1_table(struct page_in
l1_pgentry_t *pl1e;
int i;
- ASSERT(!shadow_mode_refcounts(d));
+ ASSERT(!shadow2_mode_refcounts(d));
pl1e = map_domain_page(pfn);
@@ -832,6 +834,8 @@ static int create_pae_xen_mappings(l3_pg
* 2. Cannot appear in another page table's L3:
* a. alloc_l3_table() calls this function and this check will fail
* b. mod_l3_entry() disallows updates to slot 3 in an existing table
+ *
+ * XXX -- this needs revisiting for shadow2_mode_refcount()==true...
*/
page = l3e_get_page(l3e3);
BUG_ON(page->u.inuse.type_info & PGT_pinned);
@@ -955,11 +959,7 @@ static int alloc_l2_table(struct page_in
l2_pgentry_t *pl2e;
int i;
- /* See the code in shadow_promote() to understand why this is here. */
- if ( (PGT_base_page_table == PGT_l2_page_table) &&
- unlikely(shadow_mode_refcounts(d)) )
- return 1;
- ASSERT(!shadow_mode_refcounts(d));
+ ASSERT(!shadow2_mode_refcounts(d));
pl2e = map_domain_page(pfn);
@@ -1009,11 +1009,7 @@ static int alloc_l3_table(struct page_in
l3_pgentry_t *pl3e;
int i;
- /* See the code in shadow_promote() to understand why this is here. */
- if ( (PGT_base_page_table == PGT_l3_page_table) &&
- shadow_mode_refcounts(d) )
- return 1;
- ASSERT(!shadow_mode_refcounts(d));
+ ASSERT(!shadow2_mode_refcounts(d));
#ifdef CONFIG_X86_PAE
/*
@@ -1072,11 +1068,7 @@ static int alloc_l4_table(struct page_in
unsigned long vaddr;
int i;
- /* See the code in shadow_promote() to understand why this is here. */
- if ( (PGT_base_page_table == PGT_l4_page_table) &&
- shadow_mode_refcounts(d) )
- return 1;
- ASSERT(!shadow_mode_refcounts(d));
+ ASSERT(!shadow2_mode_refcounts(d));
for ( i = 0; i < L4_PAGETABLE_ENTRIES; i++ )
{
@@ -1183,51 +1175,61 @@ static void free_l4_table(struct page_in
static inline int update_l1e(l1_pgentry_t *pl1e,
l1_pgentry_t ol1e,
- l1_pgentry_t nl1e)
-{
+ l1_pgentry_t nl1e,
+ unsigned long gl1mfn,
+ struct vcpu *v)
+{
+ int rv = 1;
+ if ( unlikely(shadow2_mode_enabled(v->domain)) )
+ shadow2_lock(v->domain);
#ifndef PTE_UPDATE_WITH_CMPXCHG
- return !__copy_to_user(pl1e, &nl1e, sizeof(nl1e));
+ rv = (!__copy_to_user(pl1e, &nl1e, sizeof(nl1e)));
#else
- intpte_t o = l1e_get_intpte(ol1e);
- intpte_t n = l1e_get_intpte(nl1e);
-
- for ( ; ; )
- {
- if ( unlikely(cmpxchg_user(pl1e, o, n) != 0) )
- {
- MEM_LOG("Failed to update %" PRIpte " -> %" PRIpte
- ": saw %" PRIpte,
- l1e_get_intpte(ol1e),
- l1e_get_intpte(nl1e),
- o);
- return 0;
- }
-
- if ( o == l1e_get_intpte(ol1e) )
- break;
-
- /* Allowed to change in Accessed/Dirty flags only. */
- BUG_ON((o ^ l1e_get_intpte(ol1e)) &
- ~(int)(_PAGE_ACCESSED|_PAGE_DIRTY));
- ol1e = l1e_from_intpte(o);
- }
-
- return 1;
+ {
+ intpte_t o = l1e_get_intpte(ol1e);
+ intpte_t n = l1e_get_intpte(nl1e);
+
+ for ( ; ; )
+ {
+ if ( unlikely(cmpxchg_user(pl1e, o, n) != 0) )
+ {
+ MEM_LOG("Failed to update %" PRIpte " -> %" PRIpte
+ ": saw %" PRIpte,
+ l1e_get_intpte(ol1e),
+ l1e_get_intpte(nl1e),
+ o);
+ rv = 0;
+ break;
+ }
+
+ if ( o == l1e_get_intpte(ol1e) )
+ break;
+
+ /* Allowed to change in Accessed/Dirty flags only. */
+ BUG_ON((o ^ l1e_get_intpte(ol1e)) &
+ ~(int)(_PAGE_ACCESSED|_PAGE_DIRTY));
+ ol1e = l1e_from_intpte(o);
+ }
+ }
#endif
+ if ( unlikely(shadow2_mode_enabled(v->domain)) )
+ {
+ shadow2_validate_guest_entry(v, _mfn(gl1mfn), pl1e);
+ shadow2_unlock(v->domain);
+ }
+ return rv;
}
/* Update the L1 entry at pl1e to new value nl1e. */
-static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e)
+static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e,
+ unsigned long gl1mfn)
{
l1_pgentry_t ol1e;
struct domain *d = current->domain;
if ( unlikely(__copy_from_user(&ol1e, pl1e, sizeof(ol1e)) != 0) )
return 0;
-
- if ( unlikely(shadow_mode_refcounts(d)) )
- return update_l1e(pl1e, ol1e, nl1e);
if ( l1e_get_flags(nl1e) & _PAGE_PRESENT )
{
@@ -1239,13 +1241,13 @@ static int mod_l1_entry(l1_pgentry_t *pl
}
/* Fast path for identical mapping, r/w and presence. */
- if ( !l1e_has_changed(ol1e, nl1e, _PAGE_RW | _PAGE_PRESENT))
- return update_l1e(pl1e, ol1e, nl1e);
+ if ( !l1e_has_changed(ol1e, nl1e, _PAGE_RW | _PAGE_PRESENT) )
+ return update_l1e(pl1e, ol1e, nl1e, gl1mfn, current);
if ( unlikely(!get_page_from_l1e(nl1e, FOREIGNDOM)) )
return 0;
- if ( unlikely(!update_l1e(pl1e, ol1e, nl1e)) )
+ if ( unlikely(!update_l1e(pl1e, ol1e, nl1e, gl1mfn, current)) )
{
put_page_from_l1e(nl1e, d);
return 0;
@@ -1253,7 +1255,7 @@ static int mod_l1_entry(l1_pgentry_t *pl
}
else
{
- if ( unlikely(!update_l1e(pl1e, ol1e, nl1e)) )
+ if ( unlikely(!update_l1e(pl1e, ol1e, nl1e, gl1mfn, current)) )
return 0;
}
@@ -1262,9 +1264,9 @@ static int mod_l1_entry(l1_pgentry_t *pl
}
#ifndef PTE_UPDATE_WITH_CMPXCHG
-#define UPDATE_ENTRY(_t,_p,_o,_n) ({ (*(_p) = (_n)); 1; })
+#define _UPDATE_ENTRY(_t,_p,_o,_n) ({ (*(_p) = (_n)); 1; })
#else
-#define UPDATE_ENTRY(_t,_p,_o,_n) ({ \
+#define _UPDATE_ENTRY(_t,_p,_o,_n) ({ \
for ( ; ; ) \
{ \
intpte_t __o = cmpxchg((intpte_t *)(_p), \
@@ -1279,6 +1281,18 @@ static int mod_l1_entry(l1_pgentry_t *pl
} \
1; })
#endif
+#define UPDATE_ENTRY(_t,_p,_o,_n,_m) ({ \
+ int rv; \
+ if ( unlikely(shadow2_mode_enabled(current->domain)) ) \
+ shadow2_lock(current->domain); \
+ rv = _UPDATE_ENTRY(_t, _p, _o, _n); \
+ if ( unlikely(shadow2_mode_enabled(current->domain)) ) \
+ { \
+ shadow2_validate_guest_entry(current, _mfn(_m), (_p)); \
+ shadow2_unlock(current->domain); \
+ } \
+ rv; \
+})
/* Update the L2 entry at pl2e to new value nl2e. pl2e is within frame pfn. */
static int mod_l2_entry(l2_pgentry_t *pl2e,
@@ -1309,19 +1323,19 @@ static int mod_l2_entry(l2_pgentry_t *pl
/* Fast path for identical mapping and presence. */
if ( !l2e_has_changed(ol2e, nl2e, _PAGE_PRESENT))
- return UPDATE_ENTRY(l2, pl2e, ol2e, nl2e);
+ return UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn);
if ( unlikely(!l1_backptr(&vaddr, pgentry_ptr_to_slot(pl2e), type)) ||
unlikely(!get_page_from_l2e(nl2e, pfn, current->domain, vaddr)) )
return 0;
- if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e)) )
+ if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn)) )
{
put_page_from_l2e(nl2e, pfn);
return 0;
}
}
- else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e)) )
+ else if ( unlikely(!UPDATE_ENTRY(l2, pl2e, ol2e, nl2e, pfn)) )
{
return 0;
}
@@ -1329,7 +1343,6 @@ static int mod_l2_entry(l2_pgentry_t *pl
put_page_from_l2e(ol2e, pfn);
return 1;
}
-
#if CONFIG_PAGING_LEVELS >= 3
@@ -1356,7 +1369,7 @@ static int mod_l3_entry(l3_pgentry_t *pl
*/
if ( pgentry_ptr_to_slot(pl3e) >= 3 )
return 0;
-#endif
+#endif
if ( unlikely(__copy_from_user(&ol3e, pl3e, sizeof(ol3e)) != 0) )
return 0;
@@ -1372,7 +1385,7 @@ static int mod_l3_entry(l3_pgentry_t *pl
/* Fast path for identical mapping and presence. */
if (!l3e_has_changed(ol3e, nl3e, _PAGE_PRESENT))
- return UPDATE_ENTRY(l3, pl3e, ol3e, nl3e);
+ return UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn);
#if CONFIG_PAGING_LEVELS >= 4
if ( unlikely(!l2_backptr(&vaddr, pgentry_ptr_to_slot(pl3e), type)) ||
@@ -1383,15 +1396,15 @@ static int mod_l3_entry(l3_pgentry_t *pl
<< L3_PAGETABLE_SHIFT;
if ( unlikely(!get_page_from_l3e(nl3e, pfn, current->domain, vaddr)) )
return 0;
-#endif
-
- if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e)) )
+#endif
+
+ if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn)) )
{
put_page_from_l3e(nl3e, pfn);
return 0;
}
}
- else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e)) )
+ else if ( unlikely(!UPDATE_ENTRY(l3, pl3e, ol3e, nl3e, pfn)) )
{
return 0;
}
@@ -1438,19 +1451,19 @@ static int mod_l4_entry(l4_pgentry_t *pl
/* Fast path for identical mapping and presence. */
if (!l4e_has_changed(ol4e, nl4e, _PAGE_PRESENT))
- return UPDATE_ENTRY(l4, pl4e, ol4e, nl4e);
+ return UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn);
if ( unlikely(!l3_backptr(&vaddr, pgentry_ptr_to_slot(pl4e), type)) ||
unlikely(!get_page_from_l4e(nl4e, pfn, current->domain, vaddr)) )
return 0;
- if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e)) )
+ if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn)) )
{
put_page_from_l4e(nl4e, pfn);
return 0;
}
}
- else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e)) )
+ else if ( unlikely(!UPDATE_ENTRY(l4, pl4e, ol4e, nl4e, pfn)) )
{
return 0;
}
@@ -1506,18 +1519,21 @@ void free_page_type(struct page_info *pa
*/
this_cpu(percpu_mm_info).deferred_ops |= DOP_FLUSH_ALL_TLBS;
- if ( unlikely(shadow_mode_enabled(owner)) )
+ if ( unlikely(shadow2_mode_enabled(owner)
+ && !shadow2_lock_is_acquired(owner)) )
{
/* Raw page tables are rewritten during save/restore. */
- if ( !shadow_mode_translate(owner) )
+ if ( !shadow2_mode_translate(owner) )
mark_dirty(owner, page_to_mfn(page));
- if ( shadow_mode_refcounts(owner) )
+ if ( shadow2_mode_refcounts(owner) )
return;
gmfn = mfn_to_gmfn(owner, page_to_mfn(page));
ASSERT(VALID_M2P(gmfn));
- remove_shadow(owner, gmfn, type & PGT_type_mask);
+ shadow2_lock(owner);
+ shadow2_remove_all_shadows(owner->vcpu[0], _mfn(gmfn));
+ shadow2_unlock(owner);
}
}
@@ -1573,9 +1589,6 @@ void put_page_type(struct page_info *pag
if ( unlikely((nx & PGT_count_mask) == 0) )
{
- /* Record TLB information for flush later. Races are harmless. */
- page->tlbflush_timestamp = tlbflush_current_time();
-
if ( unlikely((nx & PGT_type_mask) <= PGT_l4_page_table) &&
likely(nx & PGT_validated) )
{
@@ -1593,6 +1606,9 @@ void put_page_type(struct page_info *pag
x &= ~PGT_validated;
nx &= ~PGT_validated;
}
+
+ /* Record TLB information for flush later. */
+ page->tlbflush_timestamp = tlbflush_current_time();
}
else if ( unlikely((nx & (PGT_pinned|PGT_type_mask|PGT_count_mask)) ==
(PGT_pinned|PGT_l1_page_table|1)) )
@@ -1682,7 +1698,7 @@ int get_page_type(struct page_info *page
#endif
/* Fixme: add code to propagate va_unknown to subtables. */
if ( ((type & PGT_type_mask) >= PGT_l2_page_table) &&
- !shadow_mode_refcounts(page_get_owner(page)) )
+ !shadow2_mode_refcounts(page_get_owner(page)) )
return 0;
/* This table is possibly mapped at multiple locations. */
nx &= ~PGT_va_mask;
@@ -1729,7 +1745,10 @@ int new_guest_cr3(unsigned long mfn)
int okay;
unsigned long old_base_mfn;
- if ( shadow_mode_refcounts(d) )
+ if ( hvm_guest(v) && !hvm_paging_enabled(v) )
+ domain_crash_synchronous();
+
+ if ( shadow2_mode_refcounts(d) )
{
okay = get_page_from_pagenr(mfn, d);
if ( unlikely(!okay) )
@@ -1747,7 +1766,7 @@ int new_guest_cr3(unsigned long mfn)
MEM_LOG("New baseptr %lx: slow path via idle pagetables", mfn);
old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
v->arch.guest_table = pagetable_null();
- update_pagetables(v);
+ update_cr3(v);
write_cr3(__pa(idle_pg_table));
if ( old_base_mfn != 0 )
put_page_and_type(mfn_to_page(old_base_mfn));
@@ -1769,30 +1788,20 @@ int new_guest_cr3(unsigned long mfn)
invalidate_shadow_ldt(v);
old_base_mfn = pagetable_get_pfn(v->arch.guest_table);
+
v->arch.guest_table = pagetable_from_pfn(mfn);
- update_pagetables(v); /* update shadow_table and monitor_table */
+ update_cr3(v); /* update shadow_table and cr3 fields of vcpu struct */
write_ptbase(v);
if ( likely(old_base_mfn != 0) )
{
- if ( shadow_mode_refcounts(d) )
+ if ( shadow2_mode_refcounts(d) )
put_page(mfn_to_page(old_base_mfn));
else
put_page_and_type(mfn_to_page(old_base_mfn));
}
- /* CR3 also holds a ref to its shadow... */
- if ( shadow_mode_enabled(d) )
- {
- if ( v->arch.monitor_shadow_ref )
- put_shadow_ref(v->arch.monitor_shadow_ref);
- v->arch.monitor_shadow_ref =
- pagetable_get_pfn(v->arch.monitor_table);
- ASSERT(!page_get_owner(mfn_to_page(v->arch.monitor_shadow_ref)));
- get_shadow_ref(v->arch.monitor_shadow_ref);
- }
-
return 1;
}
@@ -1807,8 +1816,6 @@ static void process_deferred_ops(void)
if ( deferred_ops & (DOP_FLUSH_ALL_TLBS|DOP_FLUSH_TLB) )
{
- if ( shadow_mode_enabled(d) )
- shadow_sync_all(d);
if ( deferred_ops & DOP_FLUSH_ALL_TLBS )
flush_tlb_mask(d->domain_dirty_cpumask);
else
@@ -1974,7 +1981,7 @@ int do_mmuext_op(
type = PGT_root_page_table;
pin_page:
- if ( shadow_mode_refcounts(FOREIGNDOM) )
+ if ( shadow2_mode_refcounts(FOREIGNDOM) )
break;
okay = get_page_and_type_from_pagenr(mfn, type, FOREIGNDOM);
@@ -1996,7 +2003,7 @@ int do_mmuext_op(
break;
case MMUEXT_UNPIN_TABLE:
- if ( shadow_mode_refcounts(d) )
+ if ( shadow2_mode_refcounts(d) )
break;
if ( unlikely(!(okay = get_page_from_pagenr(mfn, d))) )
@@ -2009,6 +2016,12 @@ int do_mmuext_op(
{
put_page_and_type(page);
put_page(page);
+ if ( shadow2_mode_enabled(d) )
+ {
+ shadow2_lock(d);
+ shadow2_remove_all_shadows(v, _mfn(mfn));
+ shadow2_unlock(d);
+ }
}
else
{
@@ -2050,9 +2063,9 @@ int do_mmuext_op(
break;
case MMUEXT_INVLPG_LOCAL:
- if ( shadow_mode_enabled(d) )
- shadow_invlpg(v, op.arg1.linear_addr);
- local_flush_tlb_one(op.arg1.linear_addr);
+ if ( !shadow2_mode_enabled(d)
+ || shadow2_invlpg(v, op.arg1.linear_addr) != 0 )
+ local_flush_tlb_one(op.arg1.linear_addr);
break;
case MMUEXT_TLB_FLUSH_MULTI:
@@ -2098,7 +2111,7 @@ int do_mmuext_op(
unsigned long ptr = op.arg1.linear_addr;
unsigned long ents = op.arg2.nr_ents;
- if ( shadow_mode_external(d) )
+ if ( shadow2_mode_external(d) )
{
MEM_LOG("ignoring SET_LDT hypercall from external "
"domain %u", d->domain_id);
@@ -2171,9 +2184,6 @@ int do_mmu_update(
LOCK_BIGLOCK(d);
- if ( unlikely(shadow_mode_enabled(d)) )
- check_pagetable(v, "pre-mmu"); /* debug */
-
if ( unlikely(count & MMU_UPDATE_PREEMPTED) )
{
count &= ~MMU_UPDATE_PREEMPTED;
@@ -2248,7 +2258,12 @@ int do_mmu_update(
case PGT_l3_page_table:
case PGT_l4_page_table:
{
- ASSERT(!shadow_mode_refcounts(d));
+ if ( shadow2_mode_refcounts(d) )
+ {
+ DPRINTK("mmu update on shadow-refcounted domain!");
+ break;
+ }
+
if ( unlikely(!get_page_type(
page, type_info & (PGT_type_mask|PGT_va_mask))) )
goto not_a_pt;
@@ -2258,10 +2273,7 @@ int do_mmu_update(
case PGT_l1_page_table:
{
l1_pgentry_t l1e = l1e_from_intpte(req.val);
- okay = mod_l1_entry(va, l1e);
- if ( okay && unlikely(shadow_mode_enabled(d)) )
- shadow_l1_normal_pt_update(
- d, req.ptr, l1e, &sh_mapcache);
+ okay = mod_l1_entry(va, l1e, mfn);
}
break;
case PGT_l2_page_table:
@@ -2269,9 +2281,6 @@ int do_mmu_update(
l2_pgentry_t l2e = l2e_from_intpte(req.val);
okay = mod_l2_entry(
(l2_pgentry_t *)va, l2e, mfn, type_info);
- if ( okay && unlikely(shadow_mode_enabled(d)) )
- shadow_l2_normal_pt_update(
- d, req.ptr, l2e, &sh_mapcache);
}
break;
#if CONFIG_PAGING_LEVELS >= 3
@@ -2279,9 +2288,6 @@ int do_mmu_update(
{
l3_pgentry_t l3e = l3e_from_intpte(req.val);
okay = mod_l3_entry(va, l3e, mfn, type_info);
- if ( okay && unlikely(shadow_mode_enabled(d)) )
- shadow_l3_normal_pt_update(
- d, req.ptr, l3e, &sh_mapcache);
}
break;
#endif
@@ -2290,9 +2296,6 @@ int do_mmu_update(
{
l4_pgentry_t l4e = l4e_from_intpte(req.val);
okay = mod_l4_entry(va, l4e, mfn, type_info);
- if ( okay && unlikely(shadow_mode_enabled(d)) )
- shadow_l4_normal_pt_update(
- d, req.ptr, l4e, &sh_mapcache);
}
break;
#endif
@@ -2308,19 +2311,17 @@ int do_mmu_update(
if ( unlikely(!get_page_type(page, PGT_writable_page)) )
break;
- if ( shadow_mode_enabled(d) )
- {
- shadow_lock(d);
- __mark_dirty(d, mfn);
- if ( page_is_page_table(page) && !page_out_of_sync(page) )
- shadow_mark_mfn_out_of_sync(v, gmfn, mfn);
- }
+ if ( unlikely(shadow2_mode_enabled(d)) )
+ shadow2_lock(d);
*(intpte_t *)va = req.val;
okay = 1;
- if ( shadow_mode_enabled(d) )
- shadow_unlock(d);
+ if ( unlikely(shadow2_mode_enabled(d)) )
+ {
+ shadow2_validate_guest_entry(v, _mfn(mfn), va);
+ shadow2_unlock(d);
+ }
put_page_type(page);
}
@@ -2333,12 +2334,6 @@ int do_mmu_update(
break;
case MMU_MACHPHYS_UPDATE:
-
- if ( shadow_mode_translate(FOREIGNDOM) )
- {
- MEM_LOG("can't mutate m2p table of translate mode guest");
- break;
- }
mfn = req.ptr >> PAGE_SHIFT;
gpfn = req.val;
@@ -2349,9 +2344,13 @@ int do_mmu_update(
break;
}
- set_gpfn_from_mfn(mfn, gpfn);
+ if ( shadow2_mode_translate(FOREIGNDOM) )
+ shadow2_guest_physmap_add_page(FOREIGNDOM, gpfn, mfn);
+ else
+ set_gpfn_from_mfn(mfn, gpfn);
okay = 1;
+ // Mark the new gfn dirty...
mark_dirty(FOREIGNDOM, mfn);
put_page(mfn_to_page(mfn));
@@ -2381,9 +2380,6 @@ int do_mmu_update(
done += i;
if ( unlikely(!guest_handle_is_null(pdone)) )
copy_to_guest(pdone, &done, 1);
-
- if ( unlikely(shadow_mode_enabled(d)) )
- check_pagetable(v, "post-mmu"); /* debug */
UNLOCK_BIGLOCK(d);
return rc;
@@ -2402,7 +2398,6 @@ static int create_grant_pte_mapping(
struct domain *d = v->domain;
ASSERT(spin_is_locked(&d->big_lock));
- ASSERT(!shadow_mode_refcounts(d));
gmfn = pte_addr >> PAGE_SHIFT;
mfn = gmfn_to_mfn(d, gmfn);
@@ -2418,7 +2413,7 @@ static int create_grant_pte_mapping(
page = mfn_to_page(mfn);
type_info = page->u.inuse.type_info;
- if ( ((type_info & PGT_type_mask) != PGT_l1_page_table) ||
+ if ( ((type_info & PGT_type_mask) != PGT_l1_page_table) ||
!get_page_type(page, type_info & (PGT_type_mask|PGT_va_mask)) )
{
MEM_LOG("Grant map attempted to update a non-L1 page");
@@ -2427,28 +2422,22 @@ static int create_grant_pte_mapping(
}
ol1e = *(l1_pgentry_t *)va;
- if ( !update_l1e(va, ol1e, _nl1e) )
+ if ( !update_l1e(va, ol1e, _nl1e, mfn, v) )
{
put_page_type(page);
rc = GNTST_general_error;
goto failed;
}
- put_page_from_l1e(ol1e, d);
-
- if ( unlikely(shadow_mode_enabled(d)) )
- {
- struct domain_mmap_cache sh_mapcache;
- domain_mmap_cache_init(&sh_mapcache);
- shadow_l1_normal_pt_update(d, pte_addr, _nl1e, &sh_mapcache);
- domain_mmap_cache_destroy(&sh_mapcache);
- }
+ if ( !shadow2_mode_refcounts(d) )
+ put_page_from_l1e(ol1e, d);
put_page_type(page);
failed:
unmap_domain_page(va);
put_page(page);
+
return rc;
}
@@ -2462,8 +2451,6 @@ static int destroy_grant_pte_mapping(
u32 type_info;
l1_pgentry_t ol1e;
- ASSERT(!shadow_mode_refcounts(d));
-
gmfn = addr >> PAGE_SHIFT;
mfn = gmfn_to_mfn(d, gmfn);
@@ -2504,7 +2491,9 @@ static int destroy_grant_pte_mapping(
}
/* Delete pagetable entry. */
- if ( unlikely(!update_l1e((l1_pgentry_t *)va, ol1e, l1e_empty())) )
+ if ( unlikely(!update_l1e(
+ (l1_pgentry_t *)va, ol1e, l1e_empty(), mfn,
+ d->vcpu[0] /* Change if we go to per-vcpu shadows. */)) )
{
MEM_LOG("Cannot delete PTE entry at %p", va);
put_page_type(page);
@@ -2512,14 +2501,6 @@ static int destroy_grant_pte_mapping(
goto failed;
}
- if ( unlikely(shadow_mode_enabled(d)) )
- {
- struct domain_mmap_cache sh_mapcache;
- domain_mmap_cache_init(&sh_mapcache);
- shadow_l1_normal_pt_update(d, addr, l1e_empty(), &sh_mapcache);
- domain_mmap_cache_destroy(&sh_mapcache);
- }
-
put_page_type(page);
failed:
@@ -2536,31 +2517,22 @@ static int create_grant_va_mapping(
struct domain *d = v->domain;
ASSERT(spin_is_locked(&d->big_lock));
- ASSERT(!shadow_mode_refcounts(d));
-
- /*
- * This is actually overkill - we don't need to sync the L1 itself,
- * just everything involved in getting to this L1 (i.e. we need
- * linear_pg_table[l1_linear_offset(va)] to be in sync)...
- */
- __shadow_sync_va(v, va);
pl1e = &linear_pg_table[l1_linear_offset(va)];
if ( unlikely(__copy_from_user(&ol1e, pl1e, sizeof(ol1e)) != 0) ||
- !update_l1e(pl1e, ol1e, _nl1e) )
+ !update_l1e(pl1e, ol1e, _nl1e,
+ l2e_get_pfn(__linear_l2_table[l2_linear_offset(va)]), v) )
return GNTST_general_error;
- put_page_from_l1e(ol1e, d);
-
- if ( unlikely(shadow_mode_enabled(d)) )
- shadow_do_update_va_mapping(va, _nl1e, v);
+ if ( !shadow2_mode_refcounts(d) )
+ put_page_from_l1e(ol1e, d);
return GNTST_okay;
}
static int destroy_grant_va_mapping(
- unsigned long addr, unsigned long frame)
+ unsigned long addr, unsigned long frame, struct domain *d)
{
l1_pgentry_t *pl1e, ol1e;
@@ -2584,12 +2556,14 @@ static int destroy_grant_va_mapping(
}
/* Delete pagetable entry. */
- if ( unlikely(!update_l1e(pl1e, ol1e, l1e_empty())) )
+ if ( unlikely(!update_l1e(pl1e, ol1e, l1e_empty(),
+ l2e_get_pfn(__linear_l2_table[l2_linear_offset(addr)]),
+ d->vcpu[0] /* Change for per-vcpu shadows */)) )
{
MEM_LOG("Cannot delete PTE entry at %p", (unsigned long *)pl1e);
return GNTST_general_error;
}
-
+
return 0;
}
@@ -2597,7 +2571,7 @@ int create_grant_host_mapping(
unsigned long addr, unsigned long frame, unsigned int flags)
{
l1_pgentry_t pte = l1e_from_pfn(frame, GRANT_PTE_FLAGS);
-
+
if ( (flags & GNTMAP_application_map) )
l1e_add_flags(pte,_PAGE_USER);
if ( !(flags & GNTMAP_readonly) )
@@ -2613,7 +2587,7 @@ int destroy_grant_host_mapping(
{
if ( flags & GNTMAP_contains_pte )
return destroy_grant_pte_mapping(addr, frame, current->domain);
- return destroy_grant_va_mapping(addr, frame);
+ return destroy_grant_va_mapping(addr, frame, current->domain);
}
int steal_page(
@@ -2675,46 +2649,44 @@ int do_update_va_mapping(unsigned long v
perfc_incrc(calls_to_update_va);
- if ( unlikely(!__addr_ok(va) && !shadow_mode_external(d)) )
+ if ( unlikely(!__addr_ok(va) && !shadow2_mode_external(d)) )
return -EINVAL;
+ if ( unlikely(shadow2_mode_refcounts(d)) )
+ {
+ DPRINTK("Grant op on a shadow-refcounted domain\n");
+ return -EINVAL;
+ }
+
LOCK_BIGLOCK(d);
- if ( unlikely(shadow_mode_enabled(d)) )
- check_pagetable(v, "pre-va"); /* debug */
-
- if ( unlikely(!mod_l1_entry(&linear_pg_table[l1_linear_offset(va)],
- val)) )
- rc = -EINVAL;
-
- if ( likely(rc == 0) && unlikely(shadow_mode_enabled(d)) )
+ if ( likely(rc == 0) && unlikely(shadow2_mode_enabled(d)) )
{
if ( unlikely(this_cpu(percpu_mm_info).foreign &&
- (shadow_mode_translate(d) ||
- shadow_mode_translate(
+ (shadow2_mode_translate(d) ||
+ shadow2_mode_translate(
this_cpu(percpu_mm_info).foreign))) )
{
/*
* The foreign domain's pfn's are in a different namespace. There's
- * not enough information in just a gpte to figure out how to
+ * not enough information in just a gpte to figure out how to
* (re-)shadow this entry.
*/
domain_crash(d);
}
+ }
+
+ if ( unlikely(!mod_l1_entry(
+ &linear_pg_table[l1_linear_offset(va)], val,
+ l2e_get_pfn(__linear_l2_table[l2_linear_offset(va)]))) )
+ rc = -EINVAL;
- rc = shadow_do_update_va_mapping(va, val, v);
-
- check_pagetable(v, "post-va"); /* debug */
- }
-
switch ( flags & UVMF_FLUSHTYPE_MASK )
{
case UVMF_TLB_FLUSH:
switch ( (bmap_ptr = flags & ~UVMF_FLUSHTYPE_MASK) )
{
case UVMF_LOCAL:
- if ( unlikely(shadow_mode_enabled(d)) )
- shadow_sync_all(d);
local_flush_tlb();
break;
case UVMF_ALL:
@@ -2733,9 +2705,9 @@ int do_update_va_mapping(unsigned long v
switch ( (bmap_ptr = flags & ~UVMF_FLUSHTYPE_MASK) )
{
case UVMF_LOCAL:
- if ( unlikely(shadow_mode_enabled(d)) )
- shadow_invlpg(current, va);
- local_flush_tlb_one(va);
+ if ( !shadow2_mode_enabled(d)
+ || (shadow2_invlpg(current, va) != 0) )
+ local_flush_tlb_one(va);
break;
case UVMF_ALL:
flush_tlb_one_mask(d->domain_dirty_cpumask, va);
@@ -2807,8 +2779,6 @@ long set_gdt(struct vcpu *v,
if ( entries > FIRST_RESERVED_GDT_ENTRY )
return -EINVAL;
-
- shadow_sync_all(d);
/* Check the pages in the new GDT. */
for ( i = 0; i < nr_pages; i++ ) {
@@ -2912,24 +2882,13 @@ long do_update_descriptor(u64 pa, u64 de
break;
}
- if ( shadow_mode_enabled(dom) )
- {
- shadow_lock(dom);
-
- __mark_dirty(dom, mfn);
-
- if ( page_is_page_table(page) && !page_out_of_sync(page) )
- shadow_mark_mfn_out_of_sync(current, gmfn, mfn);
- }
+ mark_dirty(dom, mfn);
/* All is good so make the update. */
gdt_pent = map_domain_page(mfn);
memcpy(&gdt_pent[offset], &d, 8);
unmap_domain_page(gdt_pent);
- if ( shadow_mode_enabled(dom) )
- shadow_unlock(dom);
-
put_page_type(page);
ret = 0; /* success */
@@ -2981,8 +2940,8 @@ long arch_memory_op(int op, XEN_GUEST_HA
default:
break;
}
-
- if ( !shadow_mode_translate(d) || (mfn == 0) )
+
+ if ( !shadow2_mode_translate(d) || (mfn == 0) )
{
put_domain(d);
return -EINVAL;
@@ -3011,7 +2970,7 @@ long arch_memory_op(int op, XEN_GUEST_HA
guest_physmap_add_page(d, xatp.gpfn, mfn);
UNLOCK_BIGLOCK(d);
-
+
put_domain(d);
break;
@@ -3136,7 +3095,8 @@ static int ptwr_emulated_update(
unsigned long pfn;
struct page_info *page;
l1_pgentry_t pte, ol1e, nl1e, *pl1e;
- struct domain *d = current->domain;
+ struct vcpu *v = current;
+ struct domain *d = v->domain;
/* Aligned access only, thank you. */
if ( !access_ok(addr, bytes) || ((addr & (bytes-1)) != 0) )
@@ -3196,25 +3156,36 @@ static int ptwr_emulated_update(
return X86EMUL_UNHANDLEABLE;
}
+
/* Checked successfully: do the update (write or cmpxchg). */
pl1e = map_domain_page(page_to_mfn(page));
pl1e = (l1_pgentry_t *)((unsigned long)pl1e + (addr & ~PAGE_MASK));
if ( do_cmpxchg )
{
+ if ( shadow2_mode_enabled(d) )
+ shadow2_lock(d);
ol1e = l1e_from_intpte(old);
if ( cmpxchg((intpte_t *)pl1e, old, val) != old )
{
+ if ( shadow2_mode_enabled(d) )
+ shadow2_unlock(d);
unmap_domain_page(pl1e);
put_page_from_l1e(nl1e, d);
return X86EMUL_CMPXCHG_FAILED;
}
+ if ( unlikely(shadow2_mode_enabled(v->domain)) )
+ {
+ shadow2_validate_guest_entry(v, _mfn(page_to_mfn(page)), pl1e);
+ shadow2_unlock(v->domain);
+ }
}
else
{
ol1e = *pl1e;
- if ( !update_l1e(pl1e, ol1e, nl1e) )
+ if ( !update_l1e(pl1e, ol1e, nl1e, page_to_mfn(page), v) )
BUG();
}
+
unmap_domain_page(pl1e);
/* Finally, drop the old PTE. */
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/setup.c
--- a/xen/arch/x86/setup.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/setup.c Wed Aug 16 17:02:35 2006 +0100
@@ -532,8 +532,6 @@ void __init __start_xen(multiboot_info_t
if ( opt_watchdog )
watchdog_enable();
- shadow_mode_init();
-
/* initialize access control security module */
acm_init(&initrdidx, mbi, initial_images_start);
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/smpboot.c
--- a/xen/arch/x86/smpboot.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/smpboot.c Wed Aug 16 17:02:35 2006 +0100
@@ -896,7 +896,7 @@ static int __devinit do_boot_cpu(int api
v = alloc_idle_vcpu(cpu);
BUG_ON(v == NULL);
- v->arch.monitor_table = pagetable_from_paddr(__pa(idle_pg_table));
+ v->arch.cr3 = __pa(idle_pg_table);
/* start_eip had better be page-aligned! */
start_eip = setup_trampoline();
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/traps.c
--- a/xen/arch/x86/traps.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/traps.c Wed Aug 16 17:02:35 2006 +0100
@@ -277,6 +277,21 @@ void show_stack(struct cpu_user_regs *re
show_trace(regs);
}
+void show_xen_trace()
+{
+ struct cpu_user_regs regs;
+#ifdef __x86_64
+ __asm__("movq %%rsp,%0" : "=m" (regs.rsp));
+ __asm__("movq %%rbp,%0" : "=m" (regs.rbp));
+ __asm__("leaq 0(%%rip),%0" : "=a" (regs.rip));
+#else
+ __asm__("movl %%esp,%0" : "=m" (regs.esp));
+ __asm__("movl %%ebp,%0" : "=m" (regs.ebp));
+ __asm__("call 1f; 1: popl %0" : "=a" (regs.eip));
+#endif
+ show_trace(®s);
+}
+
void show_stack_overflow(unsigned long esp)
{
#ifdef MEMORY_GUARD
@@ -861,8 +876,8 @@ static int fixup_page_fault(unsigned lon
if ( unlikely(IN_HYPERVISOR_RANGE(addr)) )
{
- if ( shadow_mode_external(d) && guest_mode(regs) )
- return shadow_fault(addr, regs);
+ if ( shadow2_mode_external(d) && guest_mode(regs) )
+ return shadow2_fault(addr, regs);
if ( (addr >= GDT_LDT_VIRT_START) && (addr < GDT_LDT_VIRT_END) )
return handle_gdt_ldt_mapping_fault(
addr - GDT_LDT_VIRT_START, regs);
@@ -873,14 +888,14 @@ static int fixup_page_fault(unsigned lon
return (spurious_page_fault(addr, regs) ? EXCRET_not_a_fault : 0);
}
- if ( unlikely(shadow_mode_enabled(d)) )
- return shadow_fault(addr, regs);
-
if ( likely(VM_ASSIST(d, VMASST_TYPE_writable_pagetables)) &&
guest_kernel_mode(v, regs) &&
((regs->error_code & (PGERR_write_access|PGERR_page_present)) ==
(PGERR_write_access|PGERR_page_present)) )
return ptwr_do_page_fault(d, addr, regs) ? EXCRET_fault_fixed : 0;
+
+ if ( shadow2_mode_enabled(d) )
+ return shadow2_fault(addr, regs);
return 0;
}
@@ -905,6 +920,13 @@ asmlinkage int do_page_fault(struct cpu_
DEBUGGER_trap_entry(TRAP_page_fault, regs);
perfc_incrc(page_faults);
+
+ if ( shadow2_mode_enabled(current->domain) )
+ debugtrace_printk("%s %s %d dom=%d eip=%p cr2=%p code=%d cs=%x\n",
+ __func__, __FILE__, __LINE__,
+ current->domain->domain_id,
+ (void *)regs->eip, (void *)addr, regs->error_code,
+ regs->cs);
if ( unlikely((rc = fixup_page_fault(addr, regs)) != 0) )
return rc;
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/x86_32/domain_page.c
--- a/xen/arch/x86/x86_32/domain_page.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/x86_32/domain_page.c Wed Aug 16 17:02:35 2006 +0100
@@ -15,6 +15,7 @@
#include <asm/current.h>
#include <asm/flushtlb.h>
#include <asm/hardirq.h>
+#include <asm/hvm/support.h>
static inline struct vcpu *mapcache_current_vcpu(void)
{
@@ -58,10 +59,10 @@ void *map_domain_page(unsigned long pfn)
cache = &v->domain->arch.mapcache;
hashent = &cache->vcpu_maphash[vcpu].hash[MAPHASH_HASHFN(pfn)];
- if ( hashent->pfn == pfn )
- {
- idx = hashent->idx;
+ if ( hashent->pfn == pfn && (idx = hashent->idx) != MAPHASHENT_NOTINUSE )
+ {
hashent->refcnt++;
+ ASSERT(idx < MAPCACHE_ENTRIES);
ASSERT(hashent->refcnt != 0);
ASSERT(l1e_get_pfn(cache->l1tab[idx]) == pfn);
goto out;
@@ -178,6 +179,30 @@ void mapcache_init(struct domain *d)
MAPHASHENT_NOTINUSE;
}
+paddr_t mapped_domain_page_to_maddr(void *va)
+/* Convert a pointer in a mapped domain page to a machine address.
+ * Takes any pointer that's valid for use in unmap_domain_page() */
+{
+ unsigned int idx;
+ struct vcpu *v;
+ struct mapcache *cache;
+ unsigned long pfn;
+
+ ASSERT(!in_irq());
+
+ ASSERT((void *)MAPCACHE_VIRT_START <= va);
+ ASSERT(va < (void *)MAPCACHE_VIRT_END);
+
+ v = mapcache_current_vcpu();
+
+ cache = &v->domain->arch.mapcache;
+
+ idx = ((unsigned long)va - MAPCACHE_VIRT_START) >> PAGE_SHIFT;
+ pfn = l1e_get_pfn(cache->l1tab[idx]);
+ return ((paddr_t) pfn << PAGE_SHIFT
+ | ((unsigned long) va & ~PAGE_MASK));
+}
+
#define GLOBALMAP_BITS (IOREMAP_MBYTES << (20 - PAGE_SHIFT))
static unsigned long inuse[BITS_TO_LONGS(GLOBALMAP_BITS)];
static unsigned long garbage[BITS_TO_LONGS(GLOBALMAP_BITS)];
@@ -233,6 +258,8 @@ void unmap_domain_page_global(void *va)
l1_pgentry_t *pl1e;
unsigned int idx;
+ ASSERT((__va >= IOREMAP_VIRT_START) && (__va <= (IOREMAP_VIRT_END - 1)));
+
/* /First/, we zap the PTE. */
pl2e = virt_to_xen_l2e(__va);
pl1e = l2e_to_l1e(*pl2e) + l1_table_offset(__va);
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/x86_32/mm.c
--- a/xen/arch/x86/x86_32/mm.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/x86_32/mm.c Wed Aug 16 17:02:35 2006 +0100
@@ -75,8 +75,7 @@ void __init paging_init(void)
printk("PAE disabled.\n");
#endif
- idle_vcpu[0]->arch.monitor_table =
- pagetable_from_paddr(__pa(idle_pg_table));
+ idle_vcpu[0]->arch.cr3 = __pa(idle_pg_table);
if ( cpu_has_pge )
{
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/x86_64/mm.c
--- a/xen/arch/x86/x86_64/mm.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/x86_64/mm.c Wed Aug 16 17:02:35 2006 +0100
@@ -81,8 +81,7 @@ void __init paging_init(void)
l2_pgentry_t *l2_ro_mpt;
struct page_info *pg;
- idle_vcpu[0]->arch.monitor_table =
- pagetable_from_paddr(__pa(idle_pg_table));
+ idle_vcpu[0]->arch.cr3 = __pa(idle_pg_table);
/* Create user-accessible L2 directory to map the MPT for guests. */
l3_ro_mpt = alloc_xenheap_page();
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/x86_64/traps.c
--- a/xen/arch/x86/x86_64/traps.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/arch/x86/x86_64/traps.c Wed Aug 16 17:02:35 2006 +0100
@@ -84,7 +84,8 @@ void show_page_walk(unsigned long addr)
l4e = l4t[l4_table_offset(addr)];
mfn = l4e_get_pfn(l4e);
pfn = get_gpfn_from_mfn(mfn);
- printk(" L4 = %"PRIpte" %016lx\n", l4e_get_intpte(l4e), pfn);
+ printk(" L4[0x%lx] = %"PRIpte" %016lx\n",
+ l4_table_offset(addr), l4e_get_intpte(l4e), pfn);
if ( !(l4e_get_flags(l4e) & _PAGE_PRESENT) )
return;
@@ -92,7 +93,8 @@ void show_page_walk(unsigned long addr)
l3e = l3t[l3_table_offset(addr)];
mfn = l3e_get_pfn(l3e);
pfn = get_gpfn_from_mfn(mfn);
- printk(" L3 = %"PRIpte" %016lx\n", l3e_get_intpte(l3e), pfn);
+ printk(" L3[0x%lx] = %"PRIpte" %016lx\n",
+ l3_table_offset(addr), l3e_get_intpte(l3e), pfn);
if ( !(l3e_get_flags(l3e) & _PAGE_PRESENT) )
return;
@@ -100,7 +102,8 @@ void show_page_walk(unsigned long addr)
l2e = l2t[l2_table_offset(addr)];
mfn = l2e_get_pfn(l2e);
pfn = get_gpfn_from_mfn(mfn);
- printk(" L2 = %"PRIpte" %016lx %s\n", l2e_get_intpte(l2e), pfn,
+ printk(" L2[0x%lx] = %"PRIpte" %016lx %s\n",
+ l2_table_offset(addr), l2e_get_intpte(l2e), pfn,
(l2e_get_flags(l2e) & _PAGE_PSE) ? "(PSE)" : "");
if ( !(l2e_get_flags(l2e) & _PAGE_PRESENT) ||
(l2e_get_flags(l2e) & _PAGE_PSE) )
@@ -110,7 +113,8 @@ void show_page_walk(unsigned long addr)
l1e = l1t[l1_table_offset(addr)];
mfn = l1e_get_pfn(l1e);
pfn = get_gpfn_from_mfn(mfn);
- printk(" L1 = %"PRIpte" %016lx\n", l1e_get_intpte(l1e), pfn);
+ printk(" L1[0x%lx] = %"PRIpte" %016lx\n",
+ l1_table_offset(addr), l1e_get_intpte(l1e), pfn);
}
asmlinkage void double_fault(void);
@@ -162,7 +166,7 @@ void toggle_guest_mode(struct vcpu *v)
{
v->arch.flags ^= TF_kernel_mode;
__asm__ __volatile__ ( "swapgs" );
- update_pagetables(v);
+ update_cr3(v);
write_ptbase(v);
}
diff -r fda70200da01 -r 0f917d63e960 xen/common/acm_ops.c
--- a/xen/common/acm_ops.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/common/acm_ops.c Wed Aug 16 17:02:35 2006 +0100
@@ -26,7 +26,6 @@
#include <xen/trace.h>
#include <xen/console.h>
#include <xen/guest_access.h>
-#include <asm/shadow.h>
#include <public/sched_ctl.h>
#include <acm/acm_hooks.h>
diff -r fda70200da01 -r 0f917d63e960 xen/common/grant_table.c
--- a/xen/common/grant_table.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/common/grant_table.c Wed Aug 16 17:02:35 2006 +0100
@@ -434,7 +434,7 @@ __gnttab_unmap_grant_ref(
/* If just unmapped a writable mapping, mark as dirtied */
if ( !(flags & GNTMAP_readonly) )
- gnttab_log_dirty(rd, frame);
+ gnttab_mark_dirty(rd, frame);
if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
!(flags & GNTMAP_readonly) )
@@ -731,7 +731,7 @@ __release_grant_for_copy(
const unsigned long r_frame = act->frame;
if ( !readonly )
- gnttab_log_dirty(rd, r_frame);
+ gnttab_mark_dirty(rd, r_frame);
spin_lock(&rd->grant_table->lock);
if ( readonly )
diff -r fda70200da01 -r 0f917d63e960 xen/common/keyhandler.c
--- a/xen/common/keyhandler.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/common/keyhandler.c Wed Aug 16 17:02:35 2006 +0100
@@ -241,9 +241,6 @@ static void read_clocks(unsigned char ke
}
extern void dump_runq(unsigned char key);
-#ifndef NDEBUG
-extern void audit_domains_key(unsigned char key);
-#endif
#ifdef PERF_COUNTERS
extern void perfc_printall(unsigned char key);
@@ -261,10 +258,16 @@ static void do_debug_key(unsigned char k
#ifndef NDEBUG
static void debugtrace_key(unsigned char key)
{
- debugtrace_send_to_console = !debugtrace_send_to_console;
- debugtrace_dump();
- printk("debugtrace_printk now writing to %s.\n",
- debugtrace_send_to_console ? "console" : "buffer");
+ debugtrace_toggle();
+}
+
+static void shadow2_audit_key(unsigned char key)
+{
+ extern int shadow2_audit_enable;
+
+ shadow2_audit_enable = !shadow2_audit_enable;
+ printk("%s shadow2_audit_enable=%d\n",
+ __func__, shadow2_audit_enable);
}
#endif
@@ -288,7 +291,7 @@ void initialize_keytable(void)
#ifndef NDEBUG
register_keyhandler(
- 'o', audit_domains_key, "audit domains >0 EXPERIMENTAL");
+ 'O', shadow2_audit_key, "toggle shadow2 audits");
register_keyhandler(
'T', debugtrace_key, "toggle debugtrace to console/buffer");
#endif
diff -r fda70200da01 -r 0f917d63e960 xen/common/memory.c
--- a/xen/common/memory.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/common/memory.c Wed Aug 16 17:02:35 2006 +0100
@@ -126,6 +126,11 @@ populate_physmap(
for ( j = 0; j < (1 << extent_order); j++ )
guest_physmap_add_page(d, gpfn + j, mfn + j);
}
+ else if ( unlikely(shadow2_mode_translate(d)) )
+ {
+ for ( j = 0; j < (1 << extent_order); j++ )
+ shadow2_guest_physmap_add_page(d, gpfn + j, mfn + j);
+ }
else
{
for ( j = 0; j < (1 << extent_order); j++ )
@@ -153,7 +158,7 @@ guest_remove_page(
if ( unlikely(!mfn_valid(mfn)) )
{
DPRINTK("Domain %u page number %lx invalid\n",
- d->domain_id, mfn);
+ d->domain_id, gmfn);
return 0;
}
@@ -179,7 +184,7 @@ guest_remove_page(
(unsigned long)page->count_info, page->u.inuse.type_info);
}
- guest_physmap_remove_page(d, gmfn, mfn);
+ shadow2_guest_physmap_remove_page(d, gmfn, mfn);
put_page(page);
@@ -250,7 +255,7 @@ translate_gpfn_list(
if ( (d = find_domain_by_id(op.domid)) == NULL )
return -ESRCH;
- if ( !shadow_mode_translate(d) )
+ if ( !(shadow_mode_translate(d) || shadow2_mode_translate(d)) )
{
put_domain(d);
return -EINVAL;
diff -r fda70200da01 -r 0f917d63e960 xen/drivers/char/console.c
--- a/xen/drivers/char/console.c Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/drivers/char/console.c Wed Aug 16 17:02:35 2006 +0100
@@ -569,7 +569,7 @@ int console_getc(void)
#ifndef NDEBUG
/* Send output direct to console, or buffer it? */
-int debugtrace_send_to_console;
+static volatile int debugtrace_send_to_console;
static char *debugtrace_buf; /* Debug-trace buffer */
static unsigned int debugtrace_prd; /* Producer index */
@@ -578,16 +578,10 @@ static DEFINE_SPINLOCK(debugtrace_lock);
static DEFINE_SPINLOCK(debugtrace_lock);
integer_param("debugtrace", debugtrace_kilobytes);
-void debugtrace_dump(void)
-{
- unsigned long flags;
-
+static void debugtrace_dump_worker(void)
+{
if ( (debugtrace_bytes == 0) || !debugtrace_used )
return;
-
- watchdog_disable();
-
- spin_lock_irqsave(&debugtrace_lock, flags);
printk("debugtrace_dump() starting\n");
@@ -602,15 +596,47 @@ void debugtrace_dump(void)
memset(debugtrace_buf, '\0', debugtrace_bytes);
printk("debugtrace_dump() finished\n");
+}
+
+void debugtrace_toggle(void)
+{
+ unsigned long flags;
+
+ watchdog_disable();
+ spin_lock_irqsave(&debugtrace_lock, flags);
+
+ // dump the buffer *before* toggling, in case the act of dumping the
+ // buffer itself causes more printk's...
+ //
+ printk("debugtrace_printk now writing to %s.\n",
+ !debugtrace_send_to_console ? "console": "buffer");
+ if ( !debugtrace_send_to_console )
+ debugtrace_dump_worker();
+
+ debugtrace_send_to_console = !debugtrace_send_to_console;
spin_unlock_irqrestore(&debugtrace_lock, flags);
-
watchdog_enable();
+
+}
+
+void debugtrace_dump(void)
+{
+ unsigned long flags;
+
+ watchdog_disable();
+ spin_lock_irqsave(&debugtrace_lock, flags);
+
+ debugtrace_dump_worker();
+
+ spin_unlock_irqrestore(&debugtrace_lock, flags);
+ watchdog_enable();
}
void debugtrace_printk(const char *fmt, ...)
{
static char buf[1024];
+ static u32 count;
va_list args;
char *p;
@@ -625,8 +651,10 @@ void debugtrace_printk(const char *fmt,
ASSERT(debugtrace_buf[debugtrace_bytes - 1] == 0);
+ sprintf(buf, "%u ", ++count);
+
va_start(args, fmt);
- (void)vsnprintf(buf, sizeof(buf), fmt, args);
+ (void)vsnprintf(buf + strlen(buf), sizeof(buf), fmt, args);
va_end(args);
if ( debugtrace_send_to_console )
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/bitops.h
--- a/xen/include/asm-x86/bitops.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/bitops.h Wed Aug 16 17:02:35 2006 +0100
@@ -75,6 +75,24 @@ static __inline__ void clear_bit(int nr,
:"=m" (ADDR)
:"dIr" (nr));
}
+
+/**
+ * __clear_bit - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * Unlike clear_bit(), this function is non-atomic and may be reordered.
+ * If it's called on the same region of memory simultaneously, the effect
+ * may be that only one operation succeeds.
+ */
+static __inline__ void __clear_bit(int nr, volatile void * addr)
+{
+ __asm__(
+ "btrl %1,%0"
+ :"=m" (ADDR)
+ :"dIr" (nr));
+}
+
#define smp_mb__before_clear_bit() barrier()
#define smp_mb__after_clear_bit() barrier()
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/config.h
--- a/xen/include/asm-x86/config.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/config.h Wed Aug 16 17:02:35 2006 +0100
@@ -79,9 +79,14 @@
#ifndef __ASSEMBLY__
extern unsigned long _end; /* standard ELF symbol */
+
+static inline void FORCE_CRASH(void) __attribute__((noreturn,always_inline));
+static inline void FORCE_CRASH(void)
+{
+ __asm__ __volatile__ ( "ud2" );
+ while(1);
+}
#endif /* __ASSEMBLY__ */
-
-#define FORCE_CRASH() __asm__ __volatile__ ( "ud2" )
#if defined(__x86_64__)
@@ -149,9 +154,14 @@ extern unsigned long _end; /* standard E
/* Slot 256: read-only guest-accessible machine-to-phys translation table. */
#define RO_MPT_VIRT_START (PML4_ADDR(256))
#define RO_MPT_VIRT_END (RO_MPT_VIRT_START + PML4_ENTRY_BYTES/2)
+
+// current unused?
+#if 0
/* Slot 257: read-only guest-accessible linear page table. */
#define RO_LINEAR_PT_VIRT_START (PML4_ADDR(257))
#define RO_LINEAR_PT_VIRT_END (RO_LINEAR_PT_VIRT_START + PML4_ENTRY_BYTES)
+#endif
+
/* Slot 258: linear page table (guest table). */
#define LINEAR_PT_VIRT_START (PML4_ADDR(258))
#define LINEAR_PT_VIRT_END (LINEAR_PT_VIRT_START + PML4_ENTRY_BYTES)
@@ -175,7 +185,7 @@ extern unsigned long _end; /* standard E
#define DIRECTMAP_VIRT_START (PML4_ADDR(262))
#define DIRECTMAP_VIRT_END (DIRECTMAP_VIRT_START + PML4_ENTRY_BYTES*2)
-#define PGT_base_page_table PGT_l4_page_table
+#define PGT_base_page_table PGT_l4_page_table
#define __HYPERVISOR_CS64 0xe010
#define __HYPERVISOR_CS32 0xe008
@@ -274,9 +284,9 @@ extern unsigned long _end; /* standard E
(L2_PAGETABLE_LAST_XEN_SLOT - L2_PAGETABLE_FIRST_XEN_SLOT + 1)
#ifdef CONFIG_X86_PAE
-# define PGT_base_page_table PGT_l3_page_table
-#else
-# define PGT_base_page_table PGT_l2_page_table
+# define PGT_base_page_table PGT_l3_page_table
+#else
+# define PGT_base_page_table PGT_l2_page_table
#endif
#define __HYPERVISOR_CS 0xe008
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/domain.h
--- a/xen/include/asm-x86/domain.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/domain.h Wed Aug 16 17:02:35 2006 +0100
@@ -73,42 +73,42 @@ struct arch_domain
/* I/O-port admin-specified access capabilities. */
struct rangeset *ioport_caps;
- /* Shadow mode status and controls. */
- struct shadow_ops *ops;
- unsigned int shadow_mode; /* flags to control shadow table operation */
- unsigned int shadow_nest; /* Recursive depth of shadow_lock() nesting */
-
- /* shadow hashtable */
- struct shadow_status *shadow_ht;
- struct shadow_status *shadow_ht_free;
- struct shadow_status *shadow_ht_extras; /* extra allocation units */
- unsigned int shadow_extras_count;
-
- /* shadow dirty bitmap */
+ /* HVM stuff */
+ struct hvm_domain hvm_domain;
+
+ /* Shadow-translated guest: Pseudophys base address of reserved area. */
+ unsigned long first_reserved_pfn;
+
+ /* Shadow2 stuff */
+ u32 shadow2_mode; /* flags to control shadow operation */
+ spinlock_t shadow2_lock; /* shadow2 domain lock */
+ int shadow2_locker; /* processor which holds the lock */
+ const char *shadow2_locker_function; /* Func that took it */
+ struct list_head shadow2_freelists[SHADOW2_MAX_ORDER + 1];
+ struct list_head shadow2_p2m_freelist;
+ struct list_head shadow2_p2m_inuse;
+ struct list_head shadow2_toplevel_shadows;
+ unsigned int shadow2_total_pages; /* number of pages allocated */
+ unsigned int shadow2_free_pages; /* number of pages on freelists */
+ unsigned int shadow2_p2m_pages; /* number of pages in p2m map */
+
+ /* Shadow2 hashtable */
+ struct shadow2_hash_entry *shadow2_hash_table;
+ struct shadow2_hash_entry *shadow2_hash_freelist;
+ struct shadow2_hash_entry *shadow2_hash_allocations;
+ int shadow2_hash_walking; /* Some function is walking the hash table */
+
+ /* Shadow log-dirty bitmap */
unsigned long *shadow_dirty_bitmap;
unsigned int shadow_dirty_bitmap_size; /* in pages, bit per page */
- /* shadow mode stats */
- unsigned int shadow_page_count;
- unsigned int hl2_page_count;
- unsigned int snapshot_page_count;
-
+ /* Shadow log-dirty mode stats */
unsigned int shadow_fault_count;
unsigned int shadow_dirty_count;
- /* full shadow mode */
- struct out_of_sync_entry *out_of_sync; /* list of out-of-sync pages */
- struct out_of_sync_entry *out_of_sync_free;
- struct out_of_sync_entry *out_of_sync_extras;
- unsigned int out_of_sync_extras_count;
-
- struct list_head free_shadow_frames;
-
- pagetable_t phys_table; /* guest 1:1 pagetable */
- struct hvm_domain hvm_domain;
-
- /* Shadow-translated guest: Pseudophys base address of reserved area. */
- unsigned long first_reserved_pfn;
+ /* Shadow translated domain: P2M mapping */
+ pagetable_t phys_table;
+
} __cacheline_aligned;
#ifdef CONFIG_X86_PAE
@@ -166,25 +166,34 @@ struct arch_vcpu
*/
l1_pgentry_t *perdomain_ptes;
- pagetable_t guest_table_user; /* x86/64: user-space pagetable. */
- pagetable_t guest_table; /* (MA) guest notion of cr3 */
- pagetable_t shadow_table; /* (MA) shadow of guest */
- pagetable_t monitor_table; /* (MA) used in hypervisor */
-
- l2_pgentry_t *guest_vtable; /* virtual address of pagetable */
- l2_pgentry_t *shadow_vtable; /* virtual address of shadow_table */
- l2_pgentry_t *monitor_vtable; /* virtual address of
monitor_table */
- l1_pgentry_t *hl2_vtable; /* virtual address of hl2_table
*/
-
#ifdef CONFIG_X86_64
- l3_pgentry_t *guest_vl3table;
- l4_pgentry_t *guest_vl4table;
-#endif
-
- unsigned long monitor_shadow_ref;
+ pagetable_t guest_table_user; /* (MFN) x86/64 user-space pagetable */
+#endif
+ pagetable_t guest_table; /* (MFN) guest notion of cr3 */
+ /* guest_table holds a ref to the page, and also a type-count unless
+ * shadow refcounts are in use */
+ pagetable_t shadow_table; /* (MFN) shadow of guest */
+ pagetable_t monitor_table; /* (MFN) hypervisor PT (for HVM) */
+ unsigned long cr3; /* (MA) value to install in HW CR3
*/
+
+ void *guest_vtable; /* virtual address of pagetable */
+ void *shadow_vtable; /* virtual address of shadow_table */
+ root_pgentry_t *monitor_vtable; /* virtual address of
monitor_table */
/* Current LDT details. */
unsigned long shadow_ldt_mapcnt;
+
+ /* Shadow2 stuff */
+ /* -- pointers to mode-specific entry points */
+ struct shadow2_entry_points *shadow2;
+ unsigned long last_emulated_mfn; /* last mfn we emulated a write to */
+ u8 shadow2_propagate_fault; /* emulated fault needs to be */
+ /* propagated to guest */
+#if CONFIG_PAGING_LEVELS >= 3
+ u8 shadow2_pae_flip_pending; /* shadow update requires this PAE cpu
+ * to recopy/install its L3 table.
+ */
+#endif
} __cacheline_aligned;
/* shorthands to improve code legibility */
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/grant_table.h
--- a/xen/include/asm-x86/grant_table.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/grant_table.h Wed Aug 16 17:02:35 2006 +0100
@@ -31,7 +31,7 @@ int destroy_grant_host_mapping(
#define gnttab_shared_gmfn(d, t, i) \
(mfn_to_gmfn(d, gnttab_shared_mfn(d, t, i)))
-#define gnttab_log_dirty(d, f) mark_dirty((d), (f))
+#define gnttab_mark_dirty(d, f) mark_dirty((d), (f))
static inline void gnttab_clear_flag(unsigned long nr, uint16_t *addr)
{
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/hvm/hvm.h
--- a/xen/include/asm-x86/hvm/hvm.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/hvm/hvm.h Wed Aug 16 17:02:35 2006 +0100
@@ -56,8 +56,15 @@ struct hvm_function_table {
*/
int (*realmode)(struct vcpu *v);
int (*paging_enabled)(struct vcpu *v);
+ int (*long_mode_enabled)(struct vcpu *v);
+ int (*guest_x86_mode)(struct vcpu *v);
int (*instruction_length)(struct vcpu *v);
unsigned long (*get_guest_ctrl_reg)(struct vcpu *v, unsigned int num);
+
+ /*
+ * Re-set the value of CR3 that Xen runs on when handling VM exits
+ */
+ void (*update_host_cr3)(struct vcpu *v);
/*
* Update specifics of the guest state:
@@ -134,9 +141,27 @@ hvm_paging_enabled(struct vcpu *v)
}
static inline int
+hvm_long_mode_enabled(struct vcpu *v)
+{
+ return hvm_funcs.long_mode_enabled(v);
+}
+
+static inline int
+hvm_guest_x86_mode(struct vcpu *v)
+{
+ return hvm_funcs.guest_x86_mode(v);
+}
+
+static inline int
hvm_instruction_length(struct vcpu *v)
{
return hvm_funcs.instruction_length(v);
+}
+
+static inline void
+hvm_update_host_cr3(struct vcpu *v)
+{
+ hvm_funcs.update_host_cr3(v);
}
void hvm_hypercall_page_initialise(struct domain *d,
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/hvm/support.h
--- a/xen/include/asm-x86/hvm/support.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/hvm/support.h Wed Aug 16 17:02:35 2006 +0100
@@ -116,10 +116,13 @@ enum hval_bitmaps {
#define DBG_LEVEL_IOAPIC (1 << 9)
extern unsigned int opt_hvm_debug_level;
-#define HVM_DBG_LOG(level, _f, _a...) \
- if ( (level) & opt_hvm_debug_level ) \
- printk("[HVM:%d.%d] <%s> " _f "\n", \
- current->domain->domain_id, current->vcpu_id, __func__, ## _a)
+#define HVM_DBG_LOG(level, _f, _a...) \
+ do { \
+ if ( (level) & opt_hvm_debug_level ) \
+ printk("[HVM:%d.%d] <%s> " _f "\n", \
+ current->domain->domain_id, current->vcpu_id, __func__, \
+ ## _a); \
+ } while (0)
#else
#define HVM_DBG_LOG(level, _f, _a...)
#endif
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/hvm/vcpu.h
--- a/xen/include/asm-x86/hvm/vcpu.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/hvm/vcpu.h Wed Aug 16 17:02:35 2006 +0100
@@ -29,6 +29,7 @@
#define HVM_VCPU_INIT_SIPI_SIPI_STATE_WAIT_SIPI 1
struct hvm_vcpu {
+ unsigned long hw_cr3; /* value we give to HW to use */
unsigned long ioflags;
struct hvm_io_op io_op;
struct vlapic *vlapic;
@@ -39,6 +40,11 @@ struct hvm_vcpu {
unsigned long init_sipi_sipi_state;
int xen_port;
+
+#if CONFIG_PAGING_LEVELS >= 3
+ l3_pgentry_t hvm_lowmem_l3tab[4]
+ __attribute__((__aligned__(32)));
+#endif
/* Flags */
int flag_dr_dirty;
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/hvm/vmx/vmcs.h
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h Wed Aug 16 17:02:35 2006 +0100
@@ -87,6 +87,7 @@ struct arch_vmx_struct {
unsigned long cpu_cr0; /* copy of guest CR0 */
unsigned long cpu_shadow_cr0; /* copy of guest read shadow CR0 */
+ unsigned long cpu_shadow_cr4; /* copy of guest read shadow CR4 */
unsigned long cpu_cr2; /* save CR2 */
unsigned long cpu_cr3;
unsigned long cpu_state;
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/hvm/vmx/vmx.h
--- a/xen/include/asm-x86/hvm/vmx/vmx.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/hvm/vmx/vmx.h Wed Aug 16 17:02:35 2006 +0100
@@ -298,6 +298,9 @@ static always_inline void __vmwrite_vcpu
case GUEST_CR0:
v->arch.hvm_vmx.cpu_cr0 = value;
break;
+ case CR4_READ_SHADOW:
+ v->arch.hvm_vmx.cpu_shadow_cr4 = value;
+ break;
case CPU_BASED_VM_EXEC_CONTROL:
v->arch.hvm_vmx.cpu_based_exec_control = value;
break;
@@ -317,11 +320,14 @@ static always_inline void __vmread_vcpu(
case GUEST_CR0:
*value = v->arch.hvm_vmx.cpu_cr0;
break;
+ case CR4_READ_SHADOW:
+ *value = v->arch.hvm_vmx.cpu_shadow_cr4;
+ break;
case CPU_BASED_VM_EXEC_CONTROL:
*value = v->arch.hvm_vmx.cpu_based_exec_control;
break;
default:
- printk("__vmread_cpu: invalid field %lx\n", field);
+ printk("__vmread_vcpu: invalid field %lx\n", field);
break;
}
}
@@ -342,6 +348,7 @@ static inline int __vmwrite(unsigned lon
switch ( field ) {
case CR0_READ_SHADOW:
case GUEST_CR0:
+ case CR4_READ_SHADOW:
case CPU_BASED_VM_EXEC_CONTROL:
__vmwrite_vcpu(v, field, value);
break;
@@ -402,6 +409,46 @@ static inline int vmx_paging_enabled(str
__vmread_vcpu(v, CR0_READ_SHADOW, &cr0);
return (cr0 & X86_CR0_PE) && (cr0 & X86_CR0_PG);
+}
+
+/* Works only for vcpu == current */
+static inline int vmx_long_mode_enabled(struct vcpu *v)
+{
+ ASSERT(v == current);
+ return VMX_LONG_GUEST(current);
+}
+
+/* Works only for vcpu == current */
+static inline int vmx_realmode(struct vcpu *v)
+{
+ unsigned long rflags;
+ ASSERT(v == current);
+
+ __vmread(GUEST_RFLAGS, &rflags);
+ return rflags & X86_EFLAGS_VM;
+}
+
+/* Works only for vcpu == current */
+static inline void vmx_update_host_cr3(struct vcpu *v)
+{
+ ASSERT(v == current);
+ __vmwrite(HOST_CR3, v->arch.cr3);
+}
+
+static inline int vmx_guest_x86_mode(struct vcpu *v)
+{
+ unsigned long cs_ar_bytes;
+ ASSERT(v == current);
+
+ if ( vmx_long_mode_enabled(v) )
+ {
+ __vmread(GUEST_CS_AR_BYTES, &cs_ar_bytes);
+ return (cs_ar_bytes & (1u<<13)) ? 8 : 4;
+ }
+ if ( vmx_realmode(v) )
+ return 2;
+ __vmread(GUEST_CS_AR_BYTES, &cs_ar_bytes);
+ return (cs_ar_bytes & (1u<<14)) ? 4 : 2;
}
static inline int vmx_pgbit_test(struct vcpu *v)
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/mm.h
--- a/xen/include/asm-x86/mm.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/mm.h Wed Aug 16 17:02:35 2006 +0100
@@ -20,7 +20,11 @@ struct page_info
struct page_info
{
/* Each frame can be threaded onto a doubly-linked list. */
- struct list_head list;
+ union {
+ struct list_head list;
+ /* Shadow2 uses this field as an up-pointer in lower-level shadows */
+ paddr_t up;
+ };
/* Reference count and various PGC_xxx flags and fields. */
u32 count_info;
@@ -46,8 +50,20 @@ struct page_info
} u;
- /* Timestamp from 'TLB clock', used to reduce need for safety flushes. */
- u32 tlbflush_timestamp;
+ union {
+ /* Timestamp from 'TLB clock', used to reduce need for safety
+ * flushes. Only valid on a) free pages, and b) guest pages with a
+ * zero type count. */
+ u32 tlbflush_timestamp;
+
+ /* Only used on guest pages with a shadow.
+ * Guest pages with a shadow must have a non-zero type count, so this
+ * does not conflict with the tlbflush timestamp. */
+ u32 shadow2_flags;
+
+ // XXX -- we expect to add another field here, to be used for min/max
+ // purposes, which is only used for shadow pages.
+ };
};
/* The following page types are MUTUALLY EXCLUSIVE. */
@@ -60,6 +76,7 @@ struct page_info
#define PGT_ldt_page (6U<<29) /* using this page in an LDT? */
#define PGT_writable_page (7U<<29) /* has writable mappings of this page? */
+#ifndef SHADOW2
#define PGT_l1_shadow PGT_l1_page_table
#define PGT_l2_shadow PGT_l2_page_table
#define PGT_l3_shadow PGT_l3_page_table
@@ -69,14 +86,16 @@ struct page_info
#define PGT_writable_pred (7U<<29) /* predicted gpfn with writable ref */
#define PGT_fl1_shadow (5U<<29)
+#endif
+
#define PGT_type_mask (7U<<29) /* Bits 29-31. */
+ /* Owning guest has pinned this page to its current type? */
+#define _PGT_pinned 28
+#define PGT_pinned (1U<<_PGT_pinned)
/* Has this page been validated for use as its current type? */
-#define _PGT_validated 28
+#define _PGT_validated 27
#define PGT_validated (1U<<_PGT_validated)
- /* Owning guest has pinned this page to its current type? */
-#define _PGT_pinned 27
-#define PGT_pinned (1U<<_PGT_pinned)
#if defined(__i386__)
/* The 11 most significant bits of virt address if this is a page table. */
#define PGT_va_shift 16
@@ -98,6 +117,7 @@ struct page_info
/* 16-bit count of uses of this frame as its current type. */
#define PGT_count_mask ((1U<<16)-1)
+#ifndef SHADOW2
#ifdef __x86_64__
#define PGT_high_mfn_shift 52
#define PGT_high_mfn_mask (0xfffUL << PGT_high_mfn_shift)
@@ -112,19 +132,53 @@ struct page_info
#define PGT_score_shift 23
#define PGT_score_mask (((1U<<4)-1)<<PGT_score_shift)
#endif
+#endif /* SHADOW2 */
/* Cleared when the owning guest 'frees' this page. */
#define _PGC_allocated 31
#define PGC_allocated (1U<<_PGC_allocated)
- /* Set when fullshadow mode marks a page out-of-sync */
+ /* Set on a *guest* page to mark it out-of-sync with its shadow */
#define _PGC_out_of_sync 30
#define PGC_out_of_sync (1U<<_PGC_out_of_sync)
- /* Set when fullshadow mode is using a page as a page table */
+ /* Set when is using a page as a page table */
#define _PGC_page_table 29
#define PGC_page_table (1U<<_PGC_page_table)
/* 29-bit count of references to this frame. */
#define PGC_count_mask ((1U<<29)-1)
+/* shadow2 uses the count_info on shadow pages somewhat differently */
+/* NB: please coordinate any changes here with the SH2F's in shadow2.h */
+#define PGC_SH2_none (0U<<28) /* on the shadow2 free list */
+#define PGC_SH2_min_shadow (1U<<28)
+#define PGC_SH2_l1_32_shadow (1U<<28) /* shadowing a 32-bit L1 guest page */
+#define PGC_SH2_fl1_32_shadow (2U<<28) /* L1 shadow for a 32b 4M superpage */
+#define PGC_SH2_l2_32_shadow (3U<<28) /* shadowing a 32-bit L2 guest page */
+#define PGC_SH2_l1_pae_shadow (4U<<28) /* shadowing a pae L1 page */
+#define PGC_SH2_fl1_pae_shadow (5U<<28) /* L1 shadow for pae 2M superpg */
+#define PGC_SH2_l2_pae_shadow (6U<<28) /* shadowing a pae L2-low page */
+#define PGC_SH2_l2h_pae_shadow (7U<<28) /* shadowing a pae L2-high page */
+#define PGC_SH2_l3_pae_shadow (8U<<28) /* shadowing a pae L3 page */
+#define PGC_SH2_l1_64_shadow (9U<<28) /* shadowing a 64-bit L1 page */
+#define PGC_SH2_fl1_64_shadow (10U<<28) /* L1 shadow for 64-bit 2M superpg */
+#define PGC_SH2_l2_64_shadow (11U<<28) /* shadowing a 64-bit L2 page */
+#define PGC_SH2_l3_64_shadow (12U<<28) /* shadowing a 64-bit L3 page */
+#define PGC_SH2_l4_64_shadow (13U<<28) /* shadowing a 64-bit L4 page */
+#define PGC_SH2_max_shadow (13U<<28)
+#define PGC_SH2_p2m_table (14U<<28) /* in use as the p2m table */
+#define PGC_SH2_monitor_table (15U<<28) /* in use as a monitor table */
+#define PGC_SH2_unused (15U<<28)
+
+#define PGC_SH2_type_mask (15U<<28)
+#define PGC_SH2_type_shift 28
+
+#define PGC_SH2_pinned (1U<<27)
+
+#define _PGC_SH2_log_dirty 26
+#define PGC_SH2_log_dirty (1U<<26)
+
+/* 26 bit ref count for shadow pages */
+#define PGC_SH2_count_mask ((1U<<26) - 1)
+
/* We trust the slab allocator in slab.c, and our use of it. */
#define PageSlab(page) (1)
#define PageSetSlab(page) ((void)0)
@@ -134,14 +188,22 @@ struct page_info
#if defined(__i386__)
#define pickle_domptr(_d) ((u32)(unsigned long)(_d))
-#define unpickle_domptr(_d) ((struct domain *)(unsigned long)(_d))
+static inline struct domain *unpickle_domptr(u32 _domain)
+{ return (_domain & 1) ? NULL : (void *)_domain; }
#define PRtype_info "08lx" /* should only be used for printk's */
#elif defined(__x86_64__)
static inline struct domain *unpickle_domptr(u32 _domain)
-{ return (_domain == 0) ? NULL : __va(_domain); }
+{ return ((_domain == 0) || (_domain & 1)) ? NULL : __va(_domain); }
static inline u32 pickle_domptr(struct domain *domain)
{ return (domain == NULL) ? 0 : (u32)__pa(domain); }
#define PRtype_info "016lx"/* should only be used for printk's */
+#endif
+
+/* The order of the largest allocation unit we use for shadow pages */
+#if CONFIG_PAGING_LEVELS == 2
+#define SHADOW2_MAX_ORDER 0 /* Only ever need 4k allocations */
+#else
+#define SHADOW2_MAX_ORDER 2 /* Need up to 16k allocs for 32-bit on PAE/64 */
#endif
#define page_get_owner(_p) (unpickle_domptr((_p)->u.inuse._domain))
@@ -165,7 +227,7 @@ extern int shadow_remove_all_write_acces
extern int shadow_remove_all_write_access(
struct domain *d, unsigned long gmfn, unsigned long mfn);
extern u32 shadow_remove_all_access( struct domain *d, unsigned long gmfn);
-extern int _shadow_mode_refcounts(struct domain *d);
+extern int _shadow2_mode_refcounts(struct domain *d);
static inline void put_page(struct page_info *page)
{
@@ -197,8 +259,8 @@ static inline int get_page(struct page_i
unlikely((nx & PGC_count_mask) == 0) || /* Count overflow? */
unlikely(d != _domain) ) /* Wrong owner? */
{
- if ( !_shadow_mode_refcounts(domain) )
- DPRINTK("Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%"
+ if ( !_shadow2_mode_refcounts(domain) )
+ DPRINTK("Error pfn %lx: rd=%p, od=%p, caf=%08x, taf=%"
PRtype_info "\n",
page_to_mfn(page), domain, unpickle_domptr(d),
x, page->u.inuse.type_info);
@@ -254,6 +316,16 @@ static inline int page_is_removable(stru
ASSERT(((_p)->count_info & PGC_count_mask) != 0); \
ASSERT(page_get_owner(_p) == (_d))
+// Quick test for whether a given page can be represented directly in CR3.
+//
+#if CONFIG_PAGING_LEVELS == 3
+#define MFN_FITS_IN_CR3(_MFN) !(mfn_x(_MFN) >> 20)
+
+/* returns a lowmem machine address of the copied L3 root table */
+unsigned long
+pae_copy_root(struct vcpu *v, l3_pgentry_t *l3tab);
+#endif /* CONFIG_PAGING_LEVELS == 3 */
+
int check_descriptor(struct desc_struct *d);
/*
@@ -271,29 +343,44 @@ int check_descriptor(struct desc_struct
#define set_gpfn_from_mfn(mfn, pfn) (machine_to_phys_mapping[(mfn)] = (pfn))
#define get_gpfn_from_mfn(mfn) (machine_to_phys_mapping[(mfn)])
+
+#define mfn_to_gmfn(_d, mfn) \
+ ( (shadow2_mode_translate(_d)) \
+ ? get_gpfn_from_mfn(mfn) \
+ : (mfn) )
+
+#define gmfn_to_mfn(_d, gpfn) mfn_x(sh2_gfn_to_mfn(_d, gpfn))
+
+
/*
* The phys_to_machine_mapping is the reversed mapping of MPT for full
* virtualization. It is only used by shadow_mode_translate()==true
* guests, so we steal the address space that would have normally
* been used by the read-only MPT map.
*/
-#define phys_to_machine_mapping ((unsigned long *)RO_MPT_VIRT_START)
-#define NR_P2M_TABLE_ENTRIES ((unsigned long *)RO_MPT_VIRT_END \
- - phys_to_machine_mapping)
+#define phys_to_machine_mapping ((l1_pgentry_t *)RO_MPT_VIRT_START)
#define INVALID_MFN (~0UL)
#define VALID_MFN(_mfn) (!((_mfn) & (1U<<31)))
-#define set_mfn_from_gpfn(pfn, mfn) (phys_to_machine_mapping[(pfn)] = (mfn))
static inline unsigned long get_mfn_from_gpfn(unsigned long pfn)
{
- unsigned long mfn;
-
- if ( unlikely(pfn >= NR_P2M_TABLE_ENTRIES) ||
- unlikely(__copy_from_user(&mfn, &phys_to_machine_mapping[pfn],
- sizeof(mfn))) )
- mfn = INVALID_MFN;
-
- return mfn;
+ l1_pgentry_t l1e = l1e_empty();
+ int ret;
+
+#if CONFIG_PAGING_LEVELS > 2
+ if ( pfn > (RO_MPT_VIRT_END - RO_MPT_VIRT_START) / sizeof (l1_pgentry_t) )
+ /* This pfn is higher than the p2m map can hold */
+ return INVALID_MFN;
+#endif
+
+ ret = __copy_from_user(&l1e,
+ &phys_to_machine_mapping[pfn],
+ sizeof(l1e));
+
+ if ( (ret == 0) && (l1e_get_flags(l1e) & _PAGE_PRESENT) )
+ return l1e_get_pfn(l1e);
+
+ return INVALID_MFN;
}
#ifdef MEMORY_GUARD
@@ -333,6 +420,7 @@ void audit_domains(void);
#endif
int new_guest_cr3(unsigned long pfn);
+void make_cr3(struct vcpu *v, unsigned long mfn);
void propagate_page_fault(unsigned long addr, u16 error_code);
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/msr.h
--- a/xen/include/asm-x86/msr.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/msr.h Wed Aug 16 17:02:35 2006 +0100
@@ -112,6 +112,10 @@ static inline void wrmsrl(unsigned int m
#define MSR_IA32_VMX_EXIT_CTLS_MSR 0x483
#define MSR_IA32_VMX_ENTRY_CTLS_MSR 0x484
#define MSR_IA32_VMX_MISC_MSR 0x485
+#define MSR_IA32_VMX_CR0_FIXED0 0x486
+#define MSR_IA32_VMX_CR0_FIXED1 0x487
+#define MSR_IA32_VMX_CR4_FIXED0 0x488
+#define MSR_IA32_VMX_CR4_FIXED1 0x489
#define IA32_FEATURE_CONTROL_MSR 0x3a
#define IA32_FEATURE_CONTROL_MSR_LOCK 0x1
#define IA32_FEATURE_CONTROL_MSR_ENABLE_VMXON 0x4
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/page-guest32.h
--- a/xen/include/asm-x86/page-guest32.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/page-guest32.h Wed Aug 16 17:02:35 2006 +0100
@@ -89,15 +89,8 @@ static inline l2_pgentry_32_t l2e_from_p
#define linear_l1_table_32 \
((l1_pgentry_32_t *)(LINEAR_PT_VIRT_START))
-#define __linear_l2_table_32 \
- ((l2_pgentry_32_t *)(LINEAR_PT_VIRT_START + \
- (LINEAR_PT_VIRT_START >> (PAGETABLE_ORDER<<0))))
#define linear_pg_table_32 linear_l1_table_32
-#define linear_l2_table_32(_ed) ((_ed)->arch.guest_vtable)
-
-#define va_to_l1mfn_32(_ed, _va) \
- (l2e_get_pfn(linear_l2_table(_ed)[_va>>L2_PAGETABLE_SHIFT]))
#endif /* __X86_PAGE_GUEST_H__ */
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/page.h
--- a/xen/include/asm-x86/page.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/page.h Wed Aug 16 17:02:35 2006 +0100
@@ -233,26 +233,18 @@ typedef struct { u64 pfn; } pagetable_t;
+ DOMAIN_ENTRIES_PER_L4_PAGETABLE)
#endif
-#define LINEAR_PT_OFFSET (LINEAR_PT_VIRT_START & VADDR_MASK)
-#define linear_l1_table \
- ((l1_pgentry_t *)(LINEAR_PT_VIRT_START))
-#define __linear_l2_table \
- ((l2_pgentry_t *)(LINEAR_PT_VIRT_START + \
- (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<0))))
-#define __linear_l3_table \
- ((l3_pgentry_t *)(LINEAR_PT_VIRT_START + \
- (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<0)) + \
- (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<1))))
-#define __linear_l4_table \
- ((l4_pgentry_t *)(LINEAR_PT_VIRT_START + \
- (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<0)) + \
- (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<1)) + \
- (LINEAR_PT_OFFSET >> (PAGETABLE_ORDER<<2))))
-
+/* Where to find each level of the linear mapping */
+#define __linear_l1_table ((l1_pgentry_t *)(LINEAR_PT_VIRT_START))
+#define __linear_l2_table \
+ ((l2_pgentry_t *)(__linear_l1_table + l1_linear_offset(LINEAR_PT_VIRT_START)))
+#define __linear_l3_table \
+ ((l3_pgentry_t *)(__linear_l2_table + l2_linear_offset(LINEAR_PT_VIRT_START)))
+#define __linear_l4_table \
+ ((l4_pgentry_t *)(__linear_l3_table + l3_linear_offset(LINEAR_PT_VIRT_START)))
+
+#define linear_l1_table __linear_l1_table
#define linear_pg_table linear_l1_table
-#define linear_l2_table(v) ((v)->arch.guest_vtable)
-#define linear_l3_table(v) ((v)->arch.guest_vl3table)
-#define linear_l4_table(v) ((v)->arch.guest_vl4table)
+#define linear_l2_table(v) ((l2_pgentry_t *)(v)->arch.guest_vtable)
#ifndef __ASSEMBLY__
#if CONFIG_PAGING_LEVELS == 3
@@ -294,6 +286,7 @@ extern void paging_init(void);
#define _PAGE_AVAIL1 0x400U
#define _PAGE_AVAIL2 0x800U
#define _PAGE_AVAIL 0xE00U
+#define _PAGE_PSE_PAT 0x1000U
/*
* Debug option: Ensure that granted mappings are not implicitly unmapped.
@@ -307,9 +300,9 @@ extern void paging_init(void);
#endif
/*
- * Disallow unused flag bits plus PAT, PSE and GLOBAL. Also disallow GNTTAB
- * if we are using it for grant-table debugging. Permit the NX bit if the
- * hardware supports it.
+ * Disallow unused flag bits plus PAT, PSE and GLOBAL.
+ * Also disallow GNTTAB if we are using it for grant-table debugging.
+ * Permit the NX bit if the hardware supports it.
*/
#define BASE_DISALLOW_MASK ((0xFFFFF180U | _PAGE_GNTTAB) & ~_PAGE_NX)
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/perfc_defn.h
--- a/xen/include/asm-x86/perfc_defn.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/perfc_defn.h Wed Aug 16 17:02:35 2006 +0100
@@ -144,4 +144,57 @@ PERFCOUNTER_CPU(remove_write_bad_predict
PERFCOUNTER_CPU(remove_write_bad_prediction, "remove_write bad prediction")
PERFCOUNTER_CPU(update_hl2e_invlpg, "update_hl2e calls invlpg")
+/* Shadow2 counters */
+PERFCOUNTER_CPU(shadow2_alloc, "calls to shadow2_alloc")
+PERFCOUNTER_CPU(shadow2_alloc_tlbflush, "shadow2_alloc flushed TLBs")
+PERFSTATUS(shadow2_alloc_count, "number of shadow pages in use")
+PERFCOUNTER_CPU(shadow2_free, "calls to shadow2_free")
+PERFCOUNTER_CPU(shadow2_prealloc_1, "shadow2 recycles old shadows")
+PERFCOUNTER_CPU(shadow2_prealloc_2, "shadow2 recycles in-use shadows")
+PERFCOUNTER_CPU(shadow2_linear_map_failed, "shadow2 hit read-only linear map")
+PERFCOUNTER_CPU(shadow2_a_update, "shadow2 A bit update")
+PERFCOUNTER_CPU(shadow2_ad_update, "shadow2 A&D bit update")
+PERFCOUNTER_CPU(shadow2_fault, "calls to shadow2_fault")
+PERFCOUNTER_CPU(shadow2_fault_bail_bad_gfn, "shadow2_fault guest bad gfn")
+PERFCOUNTER_CPU(shadow2_fault_bail_not_present,
+ "shadow2_fault guest not-present")
+PERFCOUNTER_CPU(shadow2_fault_bail_nx, "shadow2_fault guest NX fault")
+PERFCOUNTER_CPU(shadow2_fault_bail_ro_mapping, "shadow2_fault guest R/W fault")
+PERFCOUNTER_CPU(shadow2_fault_bail_user_supervisor,
+ "shadow2_fault guest U/S fault")
+PERFCOUNTER_CPU(shadow2_fault_emulate_read, "shadow2_fault emulates a read")
+PERFCOUNTER_CPU(shadow2_fault_emulate_write, "shadow2_fault emulates a write")
+PERFCOUNTER_CPU(shadow2_fault_emulate_failed, "shadow2_fault emulator fails")
+PERFCOUNTER_CPU(shadow2_fault_mmio, "shadow2_fault handled as mmio")
+PERFCOUNTER_CPU(shadow2_fault_fixed, "shadow2_fault fixed fault")
+PERFCOUNTER_CPU(shadow2_ptwr_emulate, "shadow2 causes ptwr to emulate")
+PERFCOUNTER_CPU(shadow2_validate_gl1e_calls, "calls to shadow2_validate_gl1e")
+PERFCOUNTER_CPU(shadow2_validate_gl2e_calls, "calls to shadow2_validate_gl2e")
+PERFCOUNTER_CPU(shadow2_validate_gl3e_calls, "calls to shadow2_validate_gl3e")
+PERFCOUNTER_CPU(shadow2_validate_gl4e_calls, "calls to shadow2_validate_gl4e")
+PERFCOUNTER_CPU(shadow2_hash_lookups, "calls to shadow2_hash_lookup")
+PERFCOUNTER_CPU(shadow2_hash_lookup_head, "shadow2 hash hit in bucket head")
+PERFCOUNTER_CPU(shadow2_hash_lookup_miss, "shadow2 hash misses")
+PERFCOUNTER_CPU(shadow2_get_shadow_status, "calls to get_shadow_status")
+PERFCOUNTER_CPU(shadow2_hash_inserts, "calls to shadow2_hash_insert")
+PERFCOUNTER_CPU(shadow2_hash_deletes, "calls to shadow2_hash_delete")
+PERFCOUNTER_CPU(shadow2_writeable, "shadow2 removes write access")
+PERFCOUNTER_CPU(shadow2_writeable_h_1, "shadow2 writeable: 32b w2k3")
+PERFCOUNTER_CPU(shadow2_writeable_h_2, "shadow2 writeable: 32pae w2k3")
+PERFCOUNTER_CPU(shadow2_writeable_h_3, "shadow2 writeable: 64b w2k3")
+PERFCOUNTER_CPU(shadow2_writeable_h_4, "shadow2 writeable: 32b linux low")
+PERFCOUNTER_CPU(shadow2_writeable_bf, "shadow2 writeable brute-force")
+PERFCOUNTER_CPU(shadow2_mappings, "shadow2 removes all mappings")
+PERFCOUNTER_CPU(shadow2_mappings_bf, "shadow2 rm-mappings brute-force")
+PERFCOUNTER_CPU(shadow2_early_unshadow, "shadow2 unshadows for fork/exit")
+PERFCOUNTER_CPU(shadow2_early_unshadow_top, "shadow2 unhooks for fork/exit")
+PERFCOUNTER_CPU(shadow2_unshadow, "shadow2 unshadows a page")
+PERFCOUNTER_CPU(shadow2_up_pointer, "shadow2 unshadow by up-pointer")
+PERFCOUNTER_CPU(shadow2_unshadow_bf, "shadow2 unshadow brute-force")
+PERFCOUNTER_CPU(shadow2_get_page_fail, "shadow2_get_page_from_l1e failed")
+PERFCOUNTER_CPU(shadow2_guest_walk, "shadow2 walks guest tables")
+PERFCOUNTER_CPU(shadow2_walk_cache_hit, "shadow2 walk-cache hits")
+PERFCOUNTER_CPU(shadow2_walk_cache_miss, "shadow2 walk-cache misses")
+
+
/*#endif*/ /* __XEN_PERFC_DEFN_H__ */
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/processor.h
--- a/xen/include/asm-x86/processor.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/processor.h Wed Aug 16 17:02:35 2006 +0100
@@ -545,6 +545,7 @@ extern always_inline void prefetchw(cons
#endif
void show_stack(struct cpu_user_regs *regs);
+void show_xen_trace(void);
void show_stack_overflow(unsigned long esp);
void show_registers(struct cpu_user_regs *regs);
void show_execution_state(struct cpu_user_regs *regs);
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/shadow.h
--- a/xen/include/asm-x86/shadow.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/shadow.h Wed Aug 16 17:02:35 2006 +0100
@@ -1,8 +1,7 @@
/******************************************************************************
* include/asm-x86/shadow.h
*
- * Copyright (c) 2005 Michael A Fetterman
- * Based on an earlier implementation by Ian Pratt et al
+ * Copyright (c) 2006 by XenSource Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -22,1782 +21,28 @@
#ifndef _XEN_SHADOW_H
#define _XEN_SHADOW_H
-#include <xen/config.h>
-#include <xen/types.h>
-#include <xen/perfc.h>
-#include <xen/sched.h>
-#include <xen/mm.h>
-#include <xen/domain_page.h>
-#include <asm/current.h>
-#include <asm/flushtlb.h>
-#include <asm/processor.h>
-#include <asm/hvm/hvm.h>
-#include <asm/hvm/support.h>
-#include <asm/regs.h>
-#include <public/dom0_ops.h>
-#include <asm/shadow_public.h>
-#include <asm/page-guest32.h>
-#include <asm/shadow_ops.h>
+/* This file is just a wrapper around the new Shadow2 header,
+ * providing names that must be defined in any shadow implementation. */
-/* Shadow PT operation mode : shadow-mode variable in arch_domain. */
+#include <asm/shadow2.h>
-#define SHM_enable (1<<0) /* we're in one of the shadow modes */
-#define SHM_refcounts (1<<1) /* refcounts based on shadow tables instead of
- guest tables */
-#define SHM_write_all (1<<2) /* allow write access to all guest pt pages,
- regardless of pte write permissions */
-#define SHM_log_dirty (1<<3) /* enable log dirty mode */
-#define SHM_translate (1<<4) /* Xen does p2m translation, not guest */
-#define SHM_external (1<<5) /* Xen does not steal address space from the
- domain for its own booking; requires VT or
- similar mechanisms */
-#define SHM_wr_pt_pte (1<<6) /* guest allowed to set PAGE_RW bit in PTEs which
- point to page table pages. */
+/* How to make sure a page is not referred to in a shadow PT */
+/* This will need to be a for_each_vcpu if we go to per-vcpu shadows */
+#define shadow_drop_references(_d, _p) \
+ shadow2_remove_all_mappings((_d)->vcpu[0], _mfn(page_to_mfn(_p)))
+#define shadow_sync_and_drop_references(_d, _p) \
+ shadow2_remove_all_mappings((_d)->vcpu[0], _mfn(page_to_mfn(_p)))
-#define shadow_mode_enabled(_d) ((_d)->arch.shadow_mode)
-#define shadow_mode_refcounts(_d) ((_d)->arch.shadow_mode & SHM_refcounts)
-#define shadow_mode_write_l1(_d) (VM_ASSIST(_d,
VMASST_TYPE_writable_pagetables))
-#define shadow_mode_write_all(_d) ((_d)->arch.shadow_mode & SHM_write_all)
-#define shadow_mode_log_dirty(_d) ((_d)->arch.shadow_mode & SHM_log_dirty)
-#define shadow_mode_translate(_d) ((_d)->arch.shadow_mode & SHM_translate)
-#define shadow_mode_external(_d) ((_d)->arch.shadow_mode & SHM_external)
-#define shadow_mode_wr_pt_pte(_d) ((_d)->arch.shadow_mode & SHM_wr_pt_pte)
+/* Whether we are translating the domain's frame numbers for it */
+#define shadow_mode_translate(d) shadow2_mode_translate(d)
-#define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START)
-#define __shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START + \
- (SH_LINEAR_PT_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
-#define shadow_linear_l2_table(_v) ((_v)->arch.shadow_vtable)
+/* ...and if so, how to add and remove entries in the mapping */
+#define guest_physmap_add_page(_d, _p, _m) \
+ shadow2_guest_physmap_add_page((_d), (_p), (_m))
+#define guest_physmap_remove_page(_d, _p, _m ) \
+ shadow2_guest_physmap_remove_page((_d), (_p), (_m))
-// easy access to the hl2 table (for translated but not external modes only)
-#define __linear_hl2_table ((l1_pgentry_t *)(LINEAR_PT_VIRT_START + \
- (PERDOMAIN_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
-
-/*
- * For now we use the per-domain BIGLOCK rather than a shadow-specific lock.
- * We usually have the BIGLOCK already acquired anyway, so this is unlikely
- * to cause much unnecessary extra serialisation. Also it's a recursive
- * lock, and there are some code paths containing nested shadow_lock().
- * The #if0'ed code below is therefore broken until such nesting is removed.
- */
-#if 0
-#define shadow_lock_init(_d) \
- spin_lock_init(&(_d)->arch.shadow_lock)
-#define shadow_lock_is_acquired(_d) \
- spin_is_locked(&(_d)->arch.shadow_lock)
-#define shadow_lock(_d) \
-do { \
- ASSERT(!shadow_lock_is_acquired(_d)); \
- spin_lock(&(_d)->arch.shadow_lock); \
-} while (0)
-#define shadow_unlock(_d) \
-do { \
- ASSERT(!shadow_lock_is_acquired(_d)); \
- spin_unlock(&(_d)->arch.shadow_lock); \
-} while (0)
-#else
-#define shadow_lock_init(_d) \
- ((_d)->arch.shadow_nest = 0)
-#define shadow_lock_is_acquired(_d) \
- (spin_is_locked(&(_d)->big_lock) && ((_d)->arch.shadow_nest != 0))
-#define shadow_lock(_d) \
-do { \
- LOCK_BIGLOCK(_d); \
- (_d)->arch.shadow_nest++; \
-} while (0)
-#define shadow_unlock(_d) \
-do { \
- ASSERT(shadow_lock_is_acquired(_d)); \
- (_d)->arch.shadow_nest--; \
- UNLOCK_BIGLOCK(_d); \
-} while (0)
-#endif
-
-#if CONFIG_PAGING_LEVELS >= 3
-static inline u64 get_cr3_idxval(struct vcpu *v)
-{
- u64 pae_cr3;
-
- if ( v->domain->arch.ops->guest_paging_levels == PAGING_L3 &&
- !shadow_mode_log_dirty(v->domain) )
- {
- pae_cr3 = hvm_get_guest_ctrl_reg(v, 3); /* get CR3 */
- return (pae_cr3 >> PAE_CR3_ALIGN) & PAE_CR3_IDX_MASK;
- }
- else
- return 0;
-}
-
-#define shadow_key_t u64
-#define index_to_key(x) ((x) << 32)
-#else
-#define get_cr3_idxval(v) (0)
-#define shadow_key_t unsigned long
-#define index_to_key(x) (0)
-#endif
-
-
-#define SHADOW_ENCODE_MIN_MAX(_min, _max) ((((GUEST_L1_PAGETABLE_ENTRIES - 1)
- (_max)) << 16) | (_min))
-#define SHADOW_MIN(_encoded) ((_encoded) & ((1u<<16) - 1))
-#define SHADOW_MAX(_encoded) ((GUEST_L1_PAGETABLE_ENTRIES - 1) - ((_encoded)
>> 16))
-extern void shadow_direct_map_clean(struct domain *d);
-extern int shadow_direct_map_init(struct domain *d);
-extern int shadow_direct_map_fault(
- unsigned long vpa, struct cpu_user_regs *regs);
-extern void shadow_mode_init(void);
-extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc);
-extern int shadow_fault(unsigned long va, struct cpu_user_regs *regs);
-extern int shadow_mode_enable(struct domain *p, unsigned int mode);
-extern void shadow_invlpg(struct vcpu *, unsigned long);
-extern struct out_of_sync_entry *shadow_mark_mfn_out_of_sync(
- struct vcpu *v, unsigned long gpfn, unsigned long mfn);
-extern void free_monitor_pagetable(struct vcpu *v);
-extern void __shadow_sync_all(struct domain *d);
-extern int __shadow_out_of_sync(struct vcpu *v, unsigned long va);
-extern int set_p2m_entry(
- struct domain *d, unsigned long pfn, unsigned long mfn,
- struct domain_mmap_cache *l2cache,
- struct domain_mmap_cache *l1cache);
-extern void remove_shadow(struct domain *d, unsigned long gpfn, u32 stype);
-
-extern void free_shadow_page(unsigned long smfn);
-
-extern void shadow_l1_normal_pt_update(struct domain *d,
- paddr_t pa, l1_pgentry_t l1e,
- struct domain_mmap_cache *cache);
-extern void shadow_l2_normal_pt_update(struct domain *d,
- paddr_t pa, l2_pgentry_t l2e,
- struct domain_mmap_cache *cache);
-#if CONFIG_PAGING_LEVELS >= 3
-#include <asm/page-guest32.h>
-/*
- * va_mask cannot be used because it's used by the shadow hash.
- * Use the score area for for now.
- */
-#define is_xen_l2_slot(t,s)
\
- ( ((((t) & PGT_score_mask) >> PGT_score_shift) == 3) &&
\
- ((s) >= (L2_PAGETABLE_FIRST_XEN_SLOT & (L2_PAGETABLE_ENTRIES - 1))) )
-
-extern unsigned long gva_to_gpa(unsigned long gva);
-extern void shadow_l3_normal_pt_update(struct domain *d,
- paddr_t pa, l3_pgentry_t l3e,
- struct domain_mmap_cache *cache);
-#endif
-#if CONFIG_PAGING_LEVELS >= 4
-extern void shadow_l4_normal_pt_update(struct domain *d,
- paddr_t pa, l4_pgentry_t l4e,
- struct domain_mmap_cache *cache);
-#endif
-extern int shadow_do_update_va_mapping(unsigned long va,
- l1_pgentry_t val,
- struct vcpu *v);
-
-
-static inline unsigned long __shadow_status(
- struct domain *d, unsigned long gpfn, unsigned long stype);
-
-#if CONFIG_PAGING_LEVELS <= 2
-static inline void update_hl2e(struct vcpu *v, unsigned long va);
-#endif
-
-static inline int page_is_page_table(struct page_info *page)
-{
- struct domain *owner = page_get_owner(page);
- u32 type_info;
-
- if ( owner && shadow_mode_refcounts(owner) )
- return page->count_info & PGC_page_table;
-
- type_info = page->u.inuse.type_info & PGT_type_mask;
- return type_info && (type_info <= PGT_l4_page_table);
-}
-
-static inline int mfn_is_page_table(unsigned long mfn)
-{
- if ( !mfn_valid(mfn) )
- return 0;
-
- return page_is_page_table(mfn_to_page(mfn));
-}
-
-static inline int page_out_of_sync(struct page_info *page)
-{
- return page->count_info & PGC_out_of_sync;
-}
-
-static inline int mfn_out_of_sync(unsigned long mfn)
-{
- if ( !mfn_valid(mfn) )
- return 0;
-
- return page_out_of_sync(mfn_to_page(mfn));
-}
-
-
-/************************************************************************/
-
-static void inline
-__shadow_sync_mfn(struct domain *d, unsigned long mfn)
-{
- if ( d->arch.out_of_sync )
- {
- // XXX - could be smarter
- //
- __shadow_sync_all(d);
- }
-}
-
-static void inline
-__shadow_sync_va(struct vcpu *v, unsigned long va)
-{
- struct domain *d = v->domain;
-
- if ( d->arch.out_of_sync && __shadow_out_of_sync(v, va) )
- {
- perfc_incrc(shadow_sync_va);
-
- // XXX - could be smarter
- //
- __shadow_sync_all(v->domain);
- }
-#if CONFIG_PAGING_LEVELS <= 2
- // Also make sure the HL2 is up-to-date for this address.
- //
- if ( unlikely(shadow_mode_translate(v->domain)) )
- update_hl2e(v, va);
-#endif
-}
-
-static void inline
-shadow_sync_all(struct domain *d)
-{
- if ( unlikely(shadow_mode_enabled(d)) )
- {
- shadow_lock(d);
-
- if ( d->arch.out_of_sync )
- __shadow_sync_all(d);
-
- ASSERT(d->arch.out_of_sync == NULL);
-
- shadow_unlock(d);
- }
-}
-
-// SMP BUG: This routine can't ever be used properly in an SMP context.
-// It should be something like get_shadow_and_sync_va().
-// This probably shouldn't exist.
-//
-static void inline
-shadow_sync_va(struct vcpu *v, unsigned long gva)
-{
- struct domain *d = v->domain;
- if ( unlikely(shadow_mode_enabled(d)) )
- {
- shadow_lock(d);
- __shadow_sync_va(v, gva);
- shadow_unlock(d);
- }
-}
-
-extern void __shadow_mode_disable(struct domain *d);
-static inline void shadow_mode_disable(struct domain *d)
-{
- if ( unlikely(shadow_mode_enabled(d)) )
- {
- shadow_lock(d);
- __shadow_mode_disable(d);
- shadow_unlock(d);
- }
-}
-
-/************************************************************************/
-
-#define mfn_to_gmfn(_d, mfn) \
- ( (shadow_mode_translate(_d)) \
- ? get_gpfn_from_mfn(mfn) \
- : (mfn) )
-
-#define gmfn_to_mfn(_d, gpfn) \
- ({ \
- unlikely(shadow_mode_translate(_d)) \
- ? (likely(current->domain == (_d)) \
- ? get_mfn_from_gpfn(gpfn) \
- : get_mfn_from_gpfn_foreign(_d, gpfn)) \
- : (gpfn); \
- })
-
-extern unsigned long get_mfn_from_gpfn_foreign(
- struct domain *d, unsigned long gpfn);
-
-/************************************************************************/
-
-struct shadow_status {
- struct shadow_status *next; /* Pull-to-front list per hash bucket. */
- shadow_key_t gpfn_and_flags; /* Guest pfn plus flags. */
- unsigned long smfn; /* Shadow mfn. */
-};
-
-#define shadow_ht_extra_size 128
-#define shadow_ht_buckets 256
-
-struct out_of_sync_entry {
- struct out_of_sync_entry *next;
- struct vcpu *v;
- unsigned long gpfn; /* why is this here? */
- unsigned long gmfn;
- unsigned long snapshot_mfn;
- paddr_t writable_pl1e; /* NB: this is a machine address */
- unsigned long va;
-};
-
-#define out_of_sync_extra_size 127
-
-#define SHADOW_SNAPSHOT_ELSEWHERE (-1L)
-
-/************************************************************************/
-#define SHADOW_DEBUG 0
-#define SHADOW_VERBOSE_DEBUG 0
-#define SHADOW_VVERBOSE_DEBUG 0
-#define SHADOW_VVVERBOSE_DEBUG 0
-#define SHADOW_HASH_DEBUG 0
-#define FULLSHADOW_DEBUG 0
-
-#if SHADOW_DEBUG
-extern int shadow_status_noswap;
-#define SHADOW_REFLECTS_SNAPSHOT _PAGE_AVAIL0
-#endif
-
-#if SHADOW_VERBOSE_DEBUG
-#define SH_LOG(_f, _a...) \
- printk("DOM%uP%u: SH_LOG(%d): " _f "\n", \
- current->domain->domain_id , smp_processor_id(), __LINE__ , ## _a )
-#define SH_VLOG(_f, _a...) \
- printk("DOM%uP%u: SH_VLOG(%d): " _f "\n", \
- current->domain->domain_id, smp_processor_id(), __LINE__ , ## _a )
-#else
-#define SH_LOG(_f, _a...) ((void)0)
-#define SH_VLOG(_f, _a...) ((void)0)
-#endif
-
-#if SHADOW_VVERBOSE_DEBUG
-#define SH_VVLOG(_f, _a...) \
- printk("DOM%uP%u: SH_VVLOG(%d): " _f "\n", \
- current->domain->domain_id, smp_processor_id(), __LINE__ , ## _a )
-#else
-#define SH_VVLOG(_f, _a...) ((void)0)
-#endif
-
-#if SHADOW_VVVERBOSE_DEBUG
-#define SH_VVVLOG(_f, _a...) \
- printk("DOM%uP%u: SH_VVVLOG(%d): " _f "\n", \
- current->domain->domain_id, smp_processor_id(), __LINE__ , ## _a )
-#else
-#define SH_VVVLOG(_f, _a...) ((void)0)
-#endif
-
-#if FULLSHADOW_DEBUG
-#define FSH_LOG(_f, _a...) \
- printk("DOM%uP%u: FSH_LOG(%d): " _f "\n", \
- current->domain->domain_id, smp_processor_id(), __LINE__ , ## _a )
-#else
-#define FSH_LOG(_f, _a...) ((void)0)
-#endif
-
-
-/************************************************************************/
-
-static inline int
-shadow_get_page_from_l1e(l1_pgentry_t l1e, struct domain *d)
-{
- l1_pgentry_t nl1e;
- int res;
- unsigned long mfn;
- struct domain *owner;
-
- ASSERT(l1e_get_flags(l1e) & _PAGE_PRESENT);
-
- if ( !shadow_mode_refcounts(d) )
- return 1;
-
- nl1e = l1e;
- l1e_remove_flags(nl1e, _PAGE_GLOBAL);
-
- if ( unlikely(l1e_get_flags(nl1e) & L1_DISALLOW_MASK) )
- return 0;
-
- res = get_page_from_l1e(nl1e, d);
-
- if ( unlikely(!res) && IS_PRIV(d) && !shadow_mode_translate(d) &&
- !(l1e_get_flags(nl1e) & L1_DISALLOW_MASK) &&
- (mfn = l1e_get_pfn(nl1e)) &&
- mfn_valid(mfn) &&
- (owner = page_get_owner(mfn_to_page(mfn))) &&
- (d != owner) )
- {
- res = get_page_from_l1e(nl1e, owner);
- printk("tried to map mfn %lx from domain %d into shadow page tables "
- "of domain %d; %s\n",
- mfn, owner->domain_id, d->domain_id,
- res ? "success" : "failed");
- }
-
- if ( unlikely(!res) )
- {
- perfc_incrc(shadow_get_page_fail);
- FSH_LOG("%s failed to get ref l1e=%" PRIpte "\n",
- __func__, l1e_get_intpte(l1e));
- }
-
- return res;
-}
-
-static inline void
-shadow_put_page_from_l1e(l1_pgentry_t l1e, struct domain *d)
-{
- if ( !shadow_mode_refcounts(d) )
- return;
-
- put_page_from_l1e(l1e, d);
-}
-
-static inline void
-shadow_put_page_type(struct domain *d, struct page_info *page)
-{
- if ( !shadow_mode_refcounts(d) )
- return;
-
- put_page_type(page);
-}
-
-static inline int shadow_get_page(struct domain *d,
- struct page_info *page,
- struct domain *owner)
-{
- if ( !shadow_mode_refcounts(d) )
- return 1;
- return get_page(page, owner);
-}
-
-static inline void shadow_put_page(struct domain *d,
- struct page_info *page)
-{
- if ( !shadow_mode_refcounts(d) )
- return;
- put_page(page);
-}
-
-/************************************************************************/
-
-static inline void __mark_dirty(struct domain *d, unsigned long mfn)
-{
- unsigned long pfn;
-
- ASSERT(shadow_lock_is_acquired(d));
-
- if ( likely(!shadow_mode_log_dirty(d)) || !VALID_MFN(mfn) )
- return;
-
- ASSERT(d->arch.shadow_dirty_bitmap != NULL);
-
- /* We /really/ mean PFN here, even for non-translated guests. */
- pfn = get_gpfn_from_mfn(mfn);
-
- /*
- * Values with the MSB set denote MFNs that aren't really part of the
- * domain's pseudo-physical memory map (e.g., the shared info frame).
- * Nothing to do here...
- */
- if ( unlikely(IS_INVALID_M2P_ENTRY(pfn)) )
- return;
-
- /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
- if ( likely(pfn < d->arch.shadow_dirty_bitmap_size) &&
- !__test_and_set_bit(pfn, d->arch.shadow_dirty_bitmap) )
- {
- d->arch.shadow_dirty_count++;
- }
-#ifndef NDEBUG
- else if ( mfn_valid(mfn) )
- {
- SH_VLOG("mark_dirty OOR! mfn=%lx pfn=%lx max=%x (dom %p)",
- mfn, pfn, d->arch.shadow_dirty_bitmap_size, d);
- SH_VLOG("dom=%p caf=%08x taf=%" PRtype_info,
- page_get_owner(mfn_to_page(mfn)),
- mfn_to_page(mfn)->count_info,
- mfn_to_page(mfn)->u.inuse.type_info );
- }
-#endif
-}
-
-
-static inline void mark_dirty(struct domain *d, unsigned int mfn)
-{
- if ( unlikely(shadow_mode_log_dirty(d)) )
- {
- shadow_lock(d);
- __mark_dirty(d, mfn);
- shadow_unlock(d);
- }
-}
-
-
-/************************************************************************/
-#if CONFIG_PAGING_LEVELS <= 2
-static inline void
-__shadow_get_l2e(
- struct vcpu *v, unsigned long va, l2_pgentry_t *psl2e)
-{
- ASSERT(shadow_mode_enabled(v->domain));
-
- *psl2e = v->arch.shadow_vtable[l2_table_offset(va)];
-}
-
-static inline void
-__shadow_set_l2e(
- struct vcpu *v, unsigned long va, l2_pgentry_t value)
-{
- ASSERT(shadow_mode_enabled(v->domain));
-
- v->arch.shadow_vtable[l2_table_offset(va)] = value;
-}
-
-static inline void
-__guest_get_l2e(
- struct vcpu *v, unsigned long va, l2_pgentry_t *pl2e)
-{
- *pl2e = v->arch.guest_vtable[l2_table_offset(va)];
-}
-
-static inline void
-__guest_set_l2e(
- struct vcpu *v, unsigned long va, l2_pgentry_t value)
-{
- struct domain *d = v->domain;
-
- v->arch.guest_vtable[l2_table_offset(va)] = value;
-
- if ( unlikely(shadow_mode_translate(d)) )
- update_hl2e(v, va);
-
- __mark_dirty(d, pagetable_get_pfn(v->arch.guest_table));
-}
-
-static inline void
-__direct_get_l2e(
- struct vcpu *v, unsigned long va, l2_pgentry_t *psl2e)
-{
- l2_pgentry_t *phys_vtable;
-
- ASSERT(shadow_mode_enabled(v->domain));
-
- phys_vtable = map_domain_page(
- pagetable_get_pfn(v->domain->arch.phys_table));
-
- *psl2e = phys_vtable[l2_table_offset(va)];
-
- unmap_domain_page(phys_vtable);
-}
-
-static inline void
-__direct_set_l2e(
- struct vcpu *v, unsigned long va, l2_pgentry_t value)
-{
- l2_pgentry_t *phys_vtable;
-
- ASSERT(shadow_mode_enabled(v->domain));
-
- phys_vtable = map_domain_page(
- pagetable_get_pfn(v->domain->arch.phys_table));
-
- phys_vtable[l2_table_offset(va)] = value;
-
- unmap_domain_page(phys_vtable);
-}
-
-static inline void
-update_hl2e(struct vcpu *v, unsigned long va)
-{
- int index = l2_table_offset(va);
- unsigned long mfn;
- l2_pgentry_t gl2e = v->arch.guest_vtable[index];
- l1_pgentry_t old_hl2e, new_hl2e;
- int need_flush = 0;
-
- ASSERT(shadow_mode_translate(v->domain));
-
- old_hl2e = v->arch.hl2_vtable[index];
-
- if ( (l2e_get_flags(gl2e) & _PAGE_PRESENT) &&
- VALID_MFN(mfn = get_mfn_from_gpfn(l2e_get_pfn(gl2e))) )
- new_hl2e = l1e_from_pfn(mfn, __PAGE_HYPERVISOR);
- else
- new_hl2e = l1e_empty();
-
- // only do the ref counting if something has changed.
- //
- if ( (l1e_has_changed(old_hl2e, new_hl2e, PAGE_FLAG_MASK)) )
- {
- if ( (l1e_get_flags(new_hl2e) & _PAGE_PRESENT) &&
- !shadow_get_page(v->domain, mfn_to_page(l1e_get_pfn(new_hl2e)),
- v->domain) )
- new_hl2e = l1e_empty();
- if ( l1e_get_flags(old_hl2e) & _PAGE_PRESENT )
- {
- shadow_put_page(v->domain, mfn_to_page(l1e_get_pfn(old_hl2e)));
- need_flush = 1;
- }
-
- v->arch.hl2_vtable[l2_table_offset(va)] = new_hl2e;
-
- if ( need_flush )
- {
- perfc_incrc(update_hl2e_invlpg);
- flush_tlb_one_mask(v->domain->domain_dirty_cpumask,
- &linear_pg_table[l1_linear_offset(va)]);
- }
- }
-}
-
-static inline void shadow_drop_references(
- struct domain *d, struct page_info *page)
-{
- if ( likely(!shadow_mode_refcounts(d)) ||
- ((page->u.inuse.type_info & PGT_count_mask) == 0) )
- return;
-
- /* XXX This needs more thought... */
- printk("%s: needing to call shadow_remove_all_access for mfn=%lx\n",
- __func__, page_to_mfn(page));
- printk("Before: mfn=%lx c=%08x t=%" PRtype_info "\n", page_to_mfn(page),
- page->count_info, page->u.inuse.type_info);
-
- shadow_lock(d);
- shadow_remove_all_access(d, page_to_mfn(page));
- shadow_unlock(d);
-
- printk("After: mfn=%lx c=%08x t=%" PRtype_info "\n", page_to_mfn(page),
- page->count_info, page->u.inuse.type_info);
-}
-
-/* XXX Needs more thought. Neither pretty nor fast: a place holder. */
-static inline void shadow_sync_and_drop_references(
- struct domain *d, struct page_info *page)
-{
- if ( likely(!shadow_mode_refcounts(d)) )
- return;
-
- if ( page_out_of_sync(page) )
- __shadow_sync_mfn(d, page_to_mfn(page));
-
- shadow_remove_all_access(d, page_to_mfn(page));
-}
-#endif
-
-/************************************************************************/
-
-/*
- * Add another shadow reference to smfn.
- */
-static inline int
-get_shadow_ref(unsigned long smfn)
-{
- u32 x, nx;
-
- ASSERT(mfn_valid(smfn));
-
- x = mfn_to_page(smfn)->count_info;
- nx = x + 1;
-
- if ( unlikely(nx == 0) )
- {
- printk("get_shadow_ref overflow, gmfn=%" PRtype_info " smfn=%lx\n",
- mfn_to_page(smfn)->u.inuse.type_info & PGT_mfn_mask,
- smfn);
- BUG();
- }
-
- // Guarded by the shadow lock...
- //
- mfn_to_page(smfn)->count_info = nx;
-
- return 1;
-}
-
-/*
- * Drop a shadow reference to smfn.
- */
-static inline void
-put_shadow_ref(unsigned long smfn)
-{
- u32 x, nx;
-
- ASSERT(mfn_valid(smfn));
-
- x = mfn_to_page(smfn)->count_info;
- nx = x - 1;
-
- if ( unlikely(x == 0) )
- {
- printk("put_shadow_ref underflow, smfn=%lx oc=%08x t=%"
- PRtype_info "\n",
- smfn,
- mfn_to_page(smfn)->count_info,
- mfn_to_page(smfn)->u.inuse.type_info);
- BUG();
- }
-
- // Guarded by the shadow lock...
- //
- mfn_to_page(smfn)->count_info = nx;
-
- if ( unlikely(nx == 0) )
- {
- free_shadow_page(smfn);
- }
-}
-
-static inline void
-shadow_pin(unsigned long smfn)
-{
- ASSERT( !(mfn_to_page(smfn)->u.inuse.type_info & PGT_pinned) );
-
- mfn_to_page(smfn)->u.inuse.type_info |= PGT_pinned;
- if ( unlikely(!get_shadow_ref(smfn)) )
- BUG();
-}
-
-static inline void
-shadow_unpin(unsigned long smfn)
-{
- ASSERT( (mfn_to_page(smfn)->u.inuse.type_info & PGT_pinned) );
-
- mfn_to_page(smfn)->u.inuse.type_info &= ~PGT_pinned;
- put_shadow_ref(smfn);
-}
-
-/*
- * SMP issue. The following code assumes the shadow lock is held. Re-visit
- * when working on finer-gained locks for shadow.
- */
-static inline void set_guest_back_ptr(
- struct domain *d, l1_pgentry_t spte,
- unsigned long smfn, unsigned int index)
-{
- struct page_info *gpage;
-
- ASSERT(shadow_lock_is_acquired(d));
-
- if ( !shadow_mode_external(d) ||
- ((l1e_get_flags(spte) & (_PAGE_PRESENT|_PAGE_RW)) !=
- (_PAGE_PRESENT|_PAGE_RW)) )
- return;
-
- gpage = l1e_get_page(spte);
-
- ASSERT(smfn != 0);
- ASSERT(page_to_mfn(gpage) != 0);
-
- gpage->tlbflush_timestamp = smfn;
- gpage->u.inuse.type_info &= ~PGT_va_mask;
- gpage->u.inuse.type_info |= (unsigned long)index << PGT_va_shift;
-}
-
-/************************************************************************/
-#if CONFIG_PAGING_LEVELS <= 2
-extern void shadow_mark_va_out_of_sync(
- struct vcpu *v, unsigned long gpfn, unsigned long mfn,
- unsigned long va);
-
-static inline int l1pte_write_fault(
- struct vcpu *v, l1_pgentry_t *gpte_p, l1_pgentry_t *spte_p,
- unsigned long va)
-{
- struct domain *d = v->domain;
- l1_pgentry_t gpte = *gpte_p;
- l1_pgentry_t spte;
- unsigned long gpfn = l1e_get_pfn(gpte);
- unsigned long gmfn = gmfn_to_mfn(d, gpfn);
-
- //printk("l1pte_write_fault gmfn=%lx\n", gmfn);
-
- if ( unlikely(!VALID_MFN(gmfn)) )
- {
- SH_VLOG("l1pte_write_fault: invalid gpfn=%lx", gpfn);
- *spte_p = l1e_empty();
- return 0;
- }
-
- ASSERT(l1e_get_flags(gpte) & _PAGE_RW);
- l1e_add_flags(gpte, _PAGE_DIRTY | _PAGE_ACCESSED);
- spte = l1e_from_pfn(gmfn, l1e_get_flags(gpte) & ~_PAGE_GLOBAL);
-
- SH_VVLOG("l1pte_write_fault: updating spte=0x%" PRIpte " gpte=0x%" PRIpte,
- l1e_get_intpte(spte), l1e_get_intpte(gpte));
-
- __mark_dirty(d, gmfn);
-
- if ( mfn_is_page_table(gmfn) )
- shadow_mark_va_out_of_sync(v, gpfn, gmfn, va);
-
- *gpte_p = gpte;
- *spte_p = spte;
-
- return 1;
-}
-
-static inline int l1pte_read_fault(
- struct domain *d, l1_pgentry_t *gpte_p, l1_pgentry_t *spte_p)
-{
- l1_pgentry_t gpte = *gpte_p;
- l1_pgentry_t spte = *spte_p;
- unsigned long pfn = l1e_get_pfn(gpte);
- unsigned long mfn = gmfn_to_mfn(d, pfn);
-
- if ( unlikely(!VALID_MFN(mfn)) )
- {
- SH_VLOG("l1pte_read_fault: invalid gpfn=%lx", pfn);
- *spte_p = l1e_empty();
- return 0;
- }
-
- l1e_add_flags(gpte, _PAGE_ACCESSED);
- spte = l1e_from_pfn(mfn, l1e_get_flags(gpte) & ~_PAGE_GLOBAL);
-
- if ( shadow_mode_log_dirty(d) || !(l1e_get_flags(gpte) & _PAGE_DIRTY) ||
- mfn_is_page_table(mfn) )
- {
- l1e_remove_flags(spte, _PAGE_RW);
- }
-
- SH_VVLOG("l1pte_read_fault: updating spte=0x%" PRIpte " gpte=0x%" PRIpte,
- l1e_get_intpte(spte), l1e_get_intpte(gpte));
- *gpte_p = gpte;
- *spte_p = spte;
-
- return 1;
-}
-#endif
-
-static inline void l1pte_propagate_from_guest(
- struct domain *d, guest_l1_pgentry_t gpte, l1_pgentry_t *spte_p)
-{
- unsigned long mfn;
- l1_pgentry_t spte;
-
- spte = l1e_empty();
-
- if ( ((guest_l1e_get_flags(gpte) & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
- (_PAGE_PRESENT|_PAGE_ACCESSED)) &&
- VALID_MFN(mfn = gmfn_to_mfn(d, l1e_get_pfn(gpte))) )
- {
- spte = l1e_from_pfn(
- mfn, guest_l1e_get_flags(gpte) & ~(_PAGE_GLOBAL | _PAGE_AVAIL));
-
- if ( shadow_mode_log_dirty(d) ||
- !(guest_l1e_get_flags(gpte) & _PAGE_DIRTY) ||
- mfn_is_page_table(mfn) )
- {
- l1e_remove_flags(spte, _PAGE_RW);
- }
- }
-
- if ( l1e_get_intpte(spte) || l1e_get_intpte(gpte) )
- SH_VVVLOG("%s: gpte=%" PRIpte ", new spte=%" PRIpte,
- __func__, l1e_get_intpte(gpte), l1e_get_intpte(spte));
-
- *spte_p = spte;
-}
-
-static inline void hl2e_propagate_from_guest(
- struct domain *d, l2_pgentry_t gpde, l1_pgentry_t *hl2e_p)
-{
- unsigned long pfn = l2e_get_pfn(gpde);
- unsigned long mfn;
- l1_pgentry_t hl2e;
-
- hl2e = l1e_empty();
-
- if ( l2e_get_flags(gpde) & _PAGE_PRESENT )
- {
- mfn = gmfn_to_mfn(d, pfn);
- if ( VALID_MFN(mfn) && mfn_valid(mfn) )
- hl2e = l1e_from_pfn(mfn, __PAGE_HYPERVISOR);
- }
-
- if ( l1e_get_intpte(hl2e) || l2e_get_intpte(gpde) )
- SH_VVLOG("%s: gpde=%" PRIpte " hl2e=%" PRIpte, __func__,
- l2e_get_intpte(gpde), l1e_get_intpte(hl2e));
-
- *hl2e_p = hl2e;
-}
-
-static inline void l2pde_general(
- struct domain *d,
- guest_l2_pgentry_t *gpde_p,
- l2_pgentry_t *spde_p,
- unsigned long sl1mfn)
-{
- guest_l2_pgentry_t gpde = *gpde_p;
- l2_pgentry_t spde;
-
- spde = l2e_empty();
- if ( (guest_l2e_get_flags(gpde) & _PAGE_PRESENT) && (sl1mfn != 0) )
- {
- spde = l2e_from_pfn(
- sl1mfn,
- (guest_l2e_get_flags(gpde) | _PAGE_RW | _PAGE_ACCESSED) &
~_PAGE_AVAIL);
-
- /* N.B. PDEs do not have a dirty bit. */
- guest_l2e_add_flags(gpde, _PAGE_ACCESSED);
-
- *gpde_p = gpde;
- }
-
- if ( l2e_get_intpte(spde) || l2e_get_intpte(gpde) )
- SH_VVLOG("%s: gpde=%" PRIpte ", new spde=%" PRIpte, __func__,
- l2e_get_intpte(gpde), l2e_get_intpte(spde));
-
- *spde_p = spde;
-}
-
-static inline void l2pde_propagate_from_guest(
- struct domain *d, guest_l2_pgentry_t *gpde_p, l2_pgentry_t *spde_p)
-{
- guest_l2_pgentry_t gpde = *gpde_p;
- unsigned long sl1mfn = 0;
-
- if ( guest_l2e_get_flags(gpde) & _PAGE_PRESENT )
- sl1mfn = __shadow_status(d, l2e_get_pfn(gpde), PGT_l1_shadow);
- l2pde_general(d, gpde_p, spde_p, sl1mfn);
-}
-
-/************************************************************************/
-
-// returns true if a tlb flush is needed
-//
-static int inline
-validate_pte_change(
- struct domain *d,
- guest_l1_pgentry_t new_pte,
- l1_pgentry_t *shadow_pte_p)
-{
- l1_pgentry_t old_spte, new_spte;
- int need_flush = 0;
-
- perfc_incrc(validate_pte_calls);
-
- l1pte_propagate_from_guest(d, new_pte, &new_spte);
-
- if ( shadow_mode_refcounts(d) )
- {
- old_spte = *shadow_pte_p;
-
- if ( l1e_get_intpte(old_spte) == l1e_get_intpte(new_spte) )
- {
- // No accounting required...
- //
- perfc_incrc(validate_pte_changes1);
- }
- else if ( l1e_get_intpte(old_spte) ==
(l1e_get_intpte(new_spte)|_PAGE_RW) )
- {
- // Fast path for PTEs that have merely been write-protected
- // (e.g., during a Unix fork()). A strict reduction in privilege.
- //
- perfc_incrc(validate_pte_changes2);
- if ( likely(l1e_get_flags(new_spte) & _PAGE_PRESENT) )
- shadow_put_page_type(d, mfn_to_page(l1e_get_pfn(new_spte)));
- }
- else if ( ((l1e_get_flags(old_spte) | l1e_get_flags(new_spte)) &
- _PAGE_PRESENT ) &&
- l1e_has_changed(old_spte, new_spte, _PAGE_RW |
_PAGE_PRESENT) )
- {
- // only do the ref counting if something important changed.
- //
- perfc_incrc(validate_pte_changes3);
-
- if ( l1e_get_flags(old_spte) & _PAGE_PRESENT )
- {
- shadow_put_page_from_l1e(old_spte, d);
- need_flush = 1;
- }
- if ( (l1e_get_flags(new_spte) & _PAGE_PRESENT) &&
- !shadow_get_page_from_l1e(new_spte, d) ) {
- new_spte = l1e_empty();
- need_flush = -1; /* need to unshadow the page */
- }
- }
- else
- {
- perfc_incrc(validate_pte_changes4);
- }
- }
-
- *shadow_pte_p = new_spte;
-
- return need_flush;
-}
-
-// returns true if a tlb flush is needed
-//
-static int inline
-validate_hl2e_change(
- struct domain *d,
- l2_pgentry_t new_gpde,
- l1_pgentry_t *shadow_hl2e_p)
-{
- l1_pgentry_t old_hl2e, new_hl2e;
- int need_flush = 0;
-
- perfc_incrc(validate_hl2e_calls);
-
- old_hl2e = *shadow_hl2e_p;
- hl2e_propagate_from_guest(d, new_gpde, &new_hl2e);
-
- // Only do the ref counting if something important changed.
- //
- if ( ((l1e_get_flags(old_hl2e) | l1e_get_flags(new_hl2e)) & _PAGE_PRESENT)
&&
- l1e_has_changed(old_hl2e, new_hl2e, _PAGE_PRESENT) )
- {
- perfc_incrc(validate_hl2e_changes);
-
- if ( (l1e_get_flags(new_hl2e) & _PAGE_PRESENT) &&
- !get_page(mfn_to_page(l1e_get_pfn(new_hl2e)), d) )
- new_hl2e = l1e_empty();
- if ( l1e_get_flags(old_hl2e) & _PAGE_PRESENT )
- {
- put_page(mfn_to_page(l1e_get_pfn(old_hl2e)));
- need_flush = 1;
- }
- }
-
- *shadow_hl2e_p = new_hl2e;
-
- return need_flush;
-}
-
-// returns true if a tlb flush is needed
-//
-static int inline
-validate_pde_change(
- struct domain *d,
- guest_l2_pgentry_t new_gpde,
- l2_pgentry_t *shadow_pde_p)
-{
- l2_pgentry_t old_spde, new_spde;
- int need_flush = 0;
-
- perfc_incrc(validate_pde_calls);
-
- old_spde = *shadow_pde_p;
- l2pde_propagate_from_guest(d, &new_gpde, &new_spde);
-
- // Only do the ref counting if something important changed.
- //
- if ( ((l2e_get_intpte(old_spde) | l2e_get_intpte(new_spde)) &
_PAGE_PRESENT) &&
- l2e_has_changed(old_spde, new_spde, _PAGE_PRESENT) )
- {
- perfc_incrc(validate_pde_changes);
-
- if ( (l2e_get_flags(new_spde) & _PAGE_PRESENT) &&
- !get_shadow_ref(l2e_get_pfn(new_spde)) )
- BUG();
- if ( l2e_get_flags(old_spde) & _PAGE_PRESENT )
- {
- put_shadow_ref(l2e_get_pfn(old_spde));
- need_flush = 1;
- }
- }
-
- *shadow_pde_p = new_spde;
-
- return need_flush;
-}
-
-/*********************************************************************/
-
-#if SHADOW_HASH_DEBUG
-
-static void shadow_audit(struct domain *d, int print)
-{
- int live = 0, free = 0, j = 0, abs;
- struct shadow_status *a;
-
- for ( j = 0; j < shadow_ht_buckets; j++ )
- {
- a = &d->arch.shadow_ht[j];
- if ( a->gpfn_and_flags )
- {
- live++;
- ASSERT(a->smfn);
- }
- else
- ASSERT(!a->next);
-
- a = a->next;
- while ( a && (live < 9999) )
- {
- live++;
- if ( (a->gpfn_and_flags == 0) || (a->smfn == 0) )
- {
- printk("XXX live=%d gpfn+flags=%lx sp=%lx next=%p\n",
- live, a->gpfn_and_flags, a->smfn, a->next);
- BUG();
- }
- ASSERT(a->smfn);
- a = a->next;
- }
- ASSERT(live < 9999);
- }
-
- for ( a = d->arch.shadow_ht_free; a != NULL; a = a->next )
- free++;
-
- if ( print )
- printk("Xlive=%d free=%d\n", live, free);
-
- // BUG: this only works if there's only a single domain which is
- // using shadow tables.
- //
- abs = (
- perfc_value(shadow_l1_pages) +
- perfc_value(shadow_l2_pages) +
- perfc_value(hl2_table_pages) +
- perfc_value(snapshot_pages) +
- perfc_value(writable_pte_predictions)
- ) - live;
-#ifdef PERF_COUNTERS
- if ( (abs < -1) || (abs > 1) )
- {
- printk("live=%d free=%d l1=%d l2=%d hl2=%d snapshot=%d
writable_ptes=%d\n",
- live, free,
- perfc_value(shadow_l1_pages),
- perfc_value(shadow_l2_pages),
- perfc_value(hl2_table_pages),
- perfc_value(snapshot_pages),
- perfc_value(writable_pte_predictions));
- BUG();
- }
-#endif
-
- // XXX ought to add some code to audit the out-of-sync entries, too.
- //
-}
-#else
-#define shadow_audit(p, print) ((void)0)
-#endif
-
-
-static inline struct shadow_status *hash_bucket(
- struct domain *d, unsigned int gpfn)
-{
- return &d->arch.shadow_ht[gpfn % shadow_ht_buckets];
-}
-
-
-/*
- * N.B. This takes a guest pfn (i.e. a pfn in the guest's namespace,
- * which, depending on full shadow mode, may or may not equal
- * its mfn).
- * It returns the shadow's mfn, or zero if it doesn't exist.
- */
-static inline unsigned long __shadow_status(
- struct domain *d, unsigned long gpfn, unsigned long stype)
-{
- struct shadow_status *p, *x, *head;
- shadow_key_t key;
-#if CONFIG_PAGING_LEVELS >= 3
- if ( d->arch.ops->guest_paging_levels == PAGING_L3 && stype ==
PGT_l4_shadow )
- key = gpfn | stype | index_to_key(get_cr3_idxval(current));
- else
-#endif
- key = gpfn | stype;
-
- ASSERT(shadow_lock_is_acquired(d));
- ASSERT(gpfn == (gpfn & PGT_mfn_mask));
- ASSERT(stype && !(stype & ~PGT_type_mask));
-
- perfc_incrc(shadow_status_calls);
-
- x = head = hash_bucket(d, gpfn);
- p = NULL;
-
- shadow_audit(d, 0);
-
- do
- {
- ASSERT(x->gpfn_and_flags || ((x == head) && (x->next == NULL)));
-
- if ( x->gpfn_and_flags == key )
- {
-#if SHADOW_DEBUG
- if ( unlikely(shadow_status_noswap) )
- return x->smfn;
-#endif
- /* Pull-to-front if 'x' isn't already the head item. */
- if ( unlikely(x != head) )
- {
- /* Delete 'x' from list and reinsert immediately after head. */
- p->next = x->next;
- x->next = head->next;
- head->next = x;
-
- /* Swap 'x' contents with head contents. */
- SWAP(head->gpfn_and_flags, x->gpfn_and_flags);
- SWAP(head->smfn, x->smfn);
- }
- else
- {
- perfc_incrc(shadow_status_hit_head);
- }
-
- return head->smfn;
- }
-
- p = x;
- x = x->next;
- }
- while ( x != NULL );
-
- perfc_incrc(shadow_status_miss);
- return 0;
-}
-
-/*
- * Not clear if pull-to-front is worth while for this or not,
- * as it generally needs to scan the entire bucket anyway.
- * Much simpler without.
- *
- * Either returns PGT_none, or PGT_l{1,2,3,4}_page_table.
- */
-static inline u32
-shadow_max_pgtable_type(struct domain *d, unsigned long gpfn,
- unsigned long *smfn)
-{
- struct shadow_status *x;
- u32 pttype = PGT_none, type;
-
- ASSERT(shadow_lock_is_acquired(d));
- ASSERT(gpfn == (gpfn & PGT_mfn_mask));
-
- perfc_incrc(shadow_max_type);
-
- x = hash_bucket(d, gpfn);
-
- while ( x && x->gpfn_and_flags )
- {
- if ( (x->gpfn_and_flags & PGT_mfn_mask) == gpfn )
- {
- type = x->gpfn_and_flags & PGT_type_mask;
-
- switch ( type )
- {
- case PGT_hl2_shadow:
- // Treat an HL2 as if it's an L1
- //
- type = PGT_l1_shadow;
- break;
- case PGT_snapshot:
- case PGT_writable_pred:
- // Ignore snapshots -- they don't in and of themselves
constitute
- // treating a page as a page table
- //
- goto next;
- case PGT_base_page_table:
- // Early exit if we found the max possible value
- //
- return type;
- default:
- break;
- }
-
- if ( type > pttype )
- {
- pttype = type;
- if ( smfn )
- *smfn = x->smfn;
- }
- }
- next:
- x = x->next;
- }
-
- return pttype;
-}
-
-static inline void delete_shadow_status(
- struct domain *d, unsigned long gpfn, unsigned long gmfn, unsigned int
stype, u64 index)
-{
- struct shadow_status *p, *x, *n, *head;
-
- shadow_key_t key = gpfn | stype | index_to_key(index);
-
- ASSERT(shadow_lock_is_acquired(d));
- ASSERT(!(gpfn & ~PGT_mfn_mask));
- ASSERT(stype && !(stype & ~PGT_type_mask));
-
- head = hash_bucket(d, gpfn);
-
- SH_VLOG("delete gpfn=%lx t=%08x bucket=%p", gpfn, stype, head);
- shadow_audit(d, 0);
-
- /* Match on head item? */
- if ( head->gpfn_and_flags == key )
- {
- if ( (n = head->next) != NULL )
- {
- /* Overwrite head with contents of following node. */
- head->gpfn_and_flags = n->gpfn_and_flags;
- head->smfn = n->smfn;
-
- /* Delete following node. */
- head->next = n->next;
-
- /* Add deleted node to the free list. */
- n->gpfn_and_flags = 0;
- n->smfn = 0;
- n->next = d->arch.shadow_ht_free;
- d->arch.shadow_ht_free = n;
- }
- else
- {
- /* This bucket is now empty. Initialise the head node. */
- head->gpfn_and_flags = 0;
- head->smfn = 0;
- }
-
- goto found;
- }
-
- p = head;
- x = head->next;
-
- do
- {
- if ( x->gpfn_and_flags == key )
- {
- /* Delete matching node. */
- p->next = x->next;
-
- /* Add deleted node to the free list. */
- x->gpfn_and_flags = 0;
- x->smfn = 0;
- x->next = d->arch.shadow_ht_free;
- d->arch.shadow_ht_free = x;
-
- goto found;
- }
-
- p = x;
- x = x->next;
- }
- while ( x != NULL );
-
- /* If we got here, it wasn't in the list! */
- BUG();
-
- found:
- // release ref to page
- if ( stype != PGT_writable_pred )
- put_page(mfn_to_page(gmfn));
-
- shadow_audit(d, 0);
-}
-
-static inline void set_shadow_status(
- struct domain *d, unsigned long gpfn, unsigned long gmfn,
- unsigned long smfn, unsigned long stype, u64 index)
-{
- struct shadow_status *x, *head, *extra;
- int i;
-
- shadow_key_t key = gpfn | stype | index_to_key(index);
-
- SH_VVLOG("set gpfn=%lx gmfn=%lx smfn=%lx t=%lx", gpfn, gmfn, smfn, stype);
-
- ASSERT(shadow_lock_is_acquired(d));
-
- ASSERT(shadow_mode_translate(d) || gpfn);
- ASSERT(!(gpfn & ~PGT_mfn_mask));
-
- // XXX - need to be more graceful.
- ASSERT(VALID_MFN(gmfn));
-
- ASSERT(stype && !(stype & ~PGT_type_mask));
-
- x = head = hash_bucket(d, gpfn);
-
- SH_VLOG("set gpfn=%lx smfn=%lx t=%lx bucket=%p(%p)",
- gpfn, smfn, stype, x, x->next);
- shadow_audit(d, 0);
-
- // grab a reference to the guest page to represent the entry in the shadow
- // hash table
- //
- // XXX - Should PGT_writable_pred grab a page ref?
- // - Who/how are these hash table entry refs flushed if/when a page
- // is given away by the domain?
- //
- if ( stype != PGT_writable_pred )
- get_page(mfn_to_page(gmfn), d);
-
- /*
- * STEP 1. If page is already in the table, update it in place.
- */
- do
- {
- if ( unlikely(x->gpfn_and_flags == key) )
- {
- if ( stype != PGT_writable_pred )
- BUG(); // we should never replace entries into the hash table
- x->smfn = smfn;
- if ( stype != PGT_writable_pred )
- put_page(mfn_to_page(gmfn)); // already had a ref...
- goto done;
- }
-
- x = x->next;
- }
- while ( x != NULL );
-
- /*
- * STEP 2. The page must be inserted into the table.
- */
-
- /* If the bucket is empty then insert the new page as the head item. */
- if ( head->gpfn_and_flags == 0 )
- {
- head->gpfn_and_flags = key;
- head->smfn = smfn;
- ASSERT(head->next == NULL);
- goto done;
- }
-
- /* We need to allocate a new node. Ensure the quicklist is non-empty. */
- if ( unlikely(d->arch.shadow_ht_free == NULL) )
- {
- SH_VLOG("Allocate more shadow hashtable blocks.");
-
- extra = xmalloc_bytes(
- sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
-
- /* XXX Should be more graceful here. */
- if ( extra == NULL )
- BUG();
-
- memset(extra, 0, sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
-
- /* Record the allocation block so it can be correctly freed later. */
- d->arch.shadow_extras_count++;
- *((struct shadow_status **)&extra[shadow_ht_extra_size]) =
- d->arch.shadow_ht_extras;
- d->arch.shadow_ht_extras = &extra[0];
-
- /* Thread a free chain through the newly-allocated nodes. */
- for ( i = 0; i < (shadow_ht_extra_size - 1); i++ )
- extra[i].next = &extra[i+1];
- extra[i].next = NULL;
-
- /* Add the new nodes to the free list. */
- d->arch.shadow_ht_free = &extra[0];
- }
-
- /* Allocate a new node from the quicklist. */
- x = d->arch.shadow_ht_free;
- d->arch.shadow_ht_free = x->next;
-
- /* Initialise the new node and insert directly after the head item. */
- x->gpfn_and_flags = key;
- x->smfn = smfn;
- x->next = head->next;
- head->next = x;
-
- done:
- shadow_audit(d, 0);
-
- if ( stype <= PGT_l4_shadow )
- {
- // add to front of list of pages to check when removing write
- // permissions for a page...
- //
- }
-}
-
-/************************************************************************/
-
-static inline void guest_physmap_add_page(
- struct domain *d, unsigned long gpfn, unsigned long mfn)
-{
- struct domain_mmap_cache c1, c2;
-
- if ( likely(!shadow_mode_translate(d)) )
- return;
-
- domain_mmap_cache_init(&c1);
- domain_mmap_cache_init(&c2);
- shadow_lock(d);
- shadow_sync_and_drop_references(d, mfn_to_page(mfn));
- set_p2m_entry(d, gpfn, mfn, &c1, &c2);
- set_gpfn_from_mfn(mfn, gpfn);
- shadow_unlock(d);
- domain_mmap_cache_destroy(&c1);
- domain_mmap_cache_destroy(&c2);
-}
-
-static inline void guest_physmap_remove_page(
- struct domain *d, unsigned long gpfn, unsigned long mfn)
-{
- struct domain_mmap_cache c1, c2;
- unsigned long type;
-
- if ( likely(!shadow_mode_translate(d)) )
- return;
-
- domain_mmap_cache_init(&c1);
- domain_mmap_cache_init(&c2);
- shadow_lock(d);
- shadow_sync_and_drop_references(d, mfn_to_page(mfn));
- while ( (type = shadow_max_pgtable_type(d, gpfn, NULL)) != PGT_none )
- free_shadow_page(__shadow_status(d, gpfn, type));
- set_p2m_entry(d, gpfn, -1, &c1, &c2);
- set_gpfn_from_mfn(mfn, INVALID_M2P_ENTRY);
- shadow_unlock(d);
- domain_mmap_cache_destroy(&c1);
- domain_mmap_cache_destroy(&c2);
-}
-
-/************************************************************************/
-
-void static inline
-shadow_update_min_max(unsigned long smfn, int index)
-{
- struct page_info *sl1page = mfn_to_page(smfn);
- u32 min_max = sl1page->tlbflush_timestamp;
- int min = SHADOW_MIN(min_max);
- int max = SHADOW_MAX(min_max);
- int update = 0;
-
- if ( index < min )
- {
- min = index;
- update = 1;
- }
- if ( index > max )
- {
- max = index;
- update = 1;
- }
- if ( update )
- sl1page->tlbflush_timestamp = SHADOW_ENCODE_MIN_MAX(min, max);
-}
-
-#if CONFIG_PAGING_LEVELS <= 2
-extern void shadow_map_l1_into_current_l2(unsigned long va);
-
-void static inline
-shadow_set_l1e(unsigned long va, l1_pgentry_t new_spte, int create_l1_shadow)
-{
- struct vcpu *v = current;
- struct domain *d = v->domain;
- l2_pgentry_t sl2e = {0};
-
- __shadow_get_l2e(v, va, &sl2e);
- if ( !(l2e_get_flags(sl2e) & _PAGE_PRESENT) )
- {
- /*
- * Either the L1 is not shadowed, or the shadow isn't linked into
- * the current shadow L2.
- */
- if ( create_l1_shadow )
- {
- perfc_incrc(shadow_set_l1e_force_map);
- shadow_map_l1_into_current_l2(va);
- }
- else /* check to see if it exists; if so, link it in */
- {
- l2_pgentry_t gpde = linear_l2_table(v)[l2_table_offset(va)];
- unsigned long gl1pfn = l2e_get_pfn(gpde);
- unsigned long sl1mfn = __shadow_status(d, gl1pfn, PGT_l1_shadow);
-
- ASSERT( l2e_get_flags(gpde) & _PAGE_PRESENT );
-
- if ( sl1mfn )
- {
- perfc_incrc(shadow_set_l1e_unlinked);
- if ( !get_shadow_ref(sl1mfn) )
- BUG();
- l2pde_general(d, &gpde, &sl2e, sl1mfn);
- __guest_set_l2e(v, va, gpde);
- __shadow_set_l2e(v, va, sl2e);
- }
- else
- {
- // no shadow exists, so there's nothing to do.
- perfc_incrc(shadow_set_l1e_fail);
- return;
- }
- }
- }
-
- __shadow_get_l2e(v, va, &sl2e);
-
- if ( shadow_mode_refcounts(d) )
- {
- l1_pgentry_t old_spte = shadow_linear_pg_table[l1_linear_offset(va)];
-
- // only do the ref counting if something important changed.
- //
- if ( l1e_has_changed(old_spte, new_spte, _PAGE_RW | _PAGE_PRESENT) )
- {
- if ( (l1e_get_flags(new_spte) & _PAGE_PRESENT) &&
- !shadow_get_page_from_l1e(new_spte, d) )
- new_spte = l1e_empty();
- if ( l1e_get_flags(old_spte) & _PAGE_PRESENT )
- shadow_put_page_from_l1e(old_spte, d);
- }
-
- }
-
- set_guest_back_ptr(d, new_spte, l2e_get_pfn(sl2e), l1_table_offset(va));
- shadow_linear_pg_table[l1_linear_offset(va)] = new_spte;
- shadow_update_min_max(l2e_get_pfn(sl2e), l1_table_offset(va));
-}
-#endif
-/************************************************************************/
-
-static inline int
-shadow_mode_page_writable(unsigned long va, struct cpu_user_regs *regs,
unsigned long gpfn)
-{
- struct vcpu *v = current;
- struct domain *d = v->domain;
- unsigned long mfn = gmfn_to_mfn(d, gpfn);
- u32 type = mfn_to_page(mfn)->u.inuse.type_info & PGT_type_mask;
-
- if ( shadow_mode_refcounts(d) &&
- (type == PGT_writable_page) )
- type = shadow_max_pgtable_type(d, gpfn, NULL);
-
- // Strange but true: writable page tables allow kernel-mode access
- // to L1 page table pages via write-protected PTEs... Similarly, write
- // access to all page table pages is granted for shadow_mode_write_all
- // clients.
- //
- if ( ((shadow_mode_write_l1(d) && (type == PGT_l1_page_table)) ||
- (shadow_mode_write_all(d) && type && (type <= PGT_l4_page_table))) &&
- ((va < HYPERVISOR_VIRT_START)
-#if defined(__x86_64__)
- || (va >= HYPERVISOR_VIRT_END)
-#endif
- ) &&
- guest_kernel_mode(v, regs) )
- return 1;
-
- return 0;
-}
-
-#if CONFIG_PAGING_LEVELS <= 2
-static inline l1_pgentry_t gva_to_gpte(unsigned long gva)
-{
- l2_pgentry_t gpde;
- l1_pgentry_t gpte;
- struct vcpu *v = current;
-
- ASSERT( shadow_mode_translate(current->domain) );
-
- __guest_get_l2e(v, gva, &gpde);
- if ( unlikely(!(l2e_get_flags(gpde) & _PAGE_PRESENT)) )
- return l1e_empty();;
-
- // This is actually overkill - we only need to make sure the hl2
- // is in-sync.
- //
- shadow_sync_va(v, gva);
-
- if ( unlikely(__copy_from_user(&gpte,
- &linear_pg_table[gva >> PAGE_SHIFT],
- sizeof(gpte))) )
- {
- FSH_LOG("gva_to_gpte got a fault on gva=%lx", gva);
- return l1e_empty();
- }
-
- return gpte;
-}
-
-static inline unsigned long gva_to_gpa(unsigned long gva)
-{
- l1_pgentry_t gpte;
-
- gpte = gva_to_gpte(gva);
- if ( !(l1e_get_flags(gpte) & _PAGE_PRESENT) )
- return 0;
-
- return l1e_get_paddr(gpte) + (gva & ~PAGE_MASK);
-}
-#endif
-
-static inline unsigned long gva_to_mfn(unsigned long gva)
-{
- unsigned long gpa = gva_to_gpa(gva);
- return get_mfn_from_gpfn(gpa >> PAGE_SHIFT);
-}
-
-/************************************************************************/
-
-extern void __update_pagetables(struct vcpu *v);
-static inline void update_pagetables(struct vcpu *v)
-{
- struct domain *d = v->domain;
- int paging_enabled;
-
- if ( hvm_guest(v) )
- paging_enabled = hvm_paging_enabled(v);
- else
- // HACK ALERT: there's currently no easy way to figure out if a domU
- // has set its arch.guest_table to zero, vs not yet initialized it.
- //
- paging_enabled = !!pagetable_get_paddr(v->arch.guest_table);
-
- /*
- * We don't call __update_pagetables() when hvm guest paging is
- * disabled as we want the linear_pg_table to be inaccessible so that
- * we bail out early of shadow_fault() if the hvm guest tries illegal
- * accesses while it thinks paging is turned off.
- */
- if ( unlikely(shadow_mode_enabled(d)) && paging_enabled )
- {
- shadow_lock(d);
- __update_pagetables(v);
- shadow_unlock(d);
- }
-
- if ( likely(!shadow_mode_external(d)) )
- {
- if ( shadow_mode_enabled(d) )
- v->arch.monitor_table = v->arch.shadow_table;
- else
-#if CONFIG_PAGING_LEVELS == 4
- if ( !(v->arch.flags & TF_kernel_mode) )
- v->arch.monitor_table = v->arch.guest_table_user;
- else
-#endif
- v->arch.monitor_table = v->arch.guest_table;
- }
-}
-
-void clear_all_shadow_status(struct domain *d);
-
-#if SHADOW_DEBUG
-extern int _check_pagetable(struct vcpu *v, char *s);
-extern int _check_all_pagetables(struct vcpu *v, char *s);
-
-#define check_pagetable(_v, _s) _check_pagetable(_v, _s)
-//#define check_pagetable(_v, _s) _check_all_pagetables(_v, _s)
-
-#else
-#define check_pagetable(_v, _s) ((void)0)
-#endif
-
-#endif /* XEN_SHADOW_H */
+#endif /* _XEN_SHADOW_H */
/*
* Local variables:
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/x86_32/page-2level.h
--- a/xen/include/asm-x86/x86_32/page-2level.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/x86_32/page-2level.h Wed Aug 16 17:02:35 2006 +0100
@@ -46,6 +46,7 @@ typedef l2_pgentry_t root_pgentry_t;
* 12-bit flags = (pte[11:0])
*/
+#define _PAGE_NX_BIT 0U
#define _PAGE_NX 0U
/* Extract flags into 12-bit integer, or turn 12-bit flags into a pte mask. */
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/x86_32/page-3level.h
--- a/xen/include/asm-x86/x86_32/page-3level.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/x86_32/page-3level.h Wed Aug 16 17:02:35 2006 +0100
@@ -59,7 +59,8 @@ typedef l3_pgentry_t root_pgentry_t;
* 32-bit flags = (pte[63:44],pte[11:0])
*/
-#define _PAGE_NX (cpu_has_nx ? (1<<31) : 0)
+#define _PAGE_NX_BIT (1U<<31)
+#define _PAGE_NX (cpu_has_nx ? _PAGE_NX_BIT : 0)
/* Extract flags into 32-bit integer, or turn 32-bit flags into a pte mask. */
#define get_pte_flags(x) (((int)((x) >> 32) & ~0xFFF) | ((int)(x) & 0xFFF))
diff -r fda70200da01 -r 0f917d63e960 xen/include/asm-x86/x86_64/page.h
--- a/xen/include/asm-x86/x86_64/page.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/asm-x86/x86_64/page.h Wed Aug 16 17:02:35 2006 +0100
@@ -44,6 +44,8 @@ typedef l4_pgentry_t root_pgentry_t;
/* Given a virtual address, get an entry offset into a linear page table. */
#define l1_linear_offset(_a) (((_a) & VADDR_MASK) >> L1_PAGETABLE_SHIFT)
#define l2_linear_offset(_a) (((_a) & VADDR_MASK) >> L2_PAGETABLE_SHIFT)
+#define l3_linear_offset(_a) (((_a) & VADDR_MASK) >> L3_PAGETABLE_SHIFT)
+#define l4_linear_offset(_a) (((_a) & VADDR_MASK) >> L4_PAGETABLE_SHIFT)
#define is_guest_l1_slot(_s) (1)
#define is_guest_l2_slot(_t, _s) (1)
@@ -70,7 +72,8 @@ typedef l4_pgentry_t root_pgentry_t;
#define put_pte_flags(x) (((intpte_t)((x) & ~0xFFF) << 40) | ((x) & 0xFFF))
/* Bit 23 of a 24-bit flag mask. This corresponds to bit 63 of a pte.*/
-#define _PAGE_NX (cpu_has_nx ? (1U<<23) : 0U)
+#define _PAGE_NX_BIT (1U<<23)
+#define _PAGE_NX (cpu_has_nx ? _PAGE_NX_BIT : 0U)
#define L1_DISALLOW_MASK BASE_DISALLOW_MASK
#define L2_DISALLOW_MASK BASE_DISALLOW_MASK
diff -r fda70200da01 -r 0f917d63e960 xen/include/public/dom0_ops.h
--- a/xen/include/public/dom0_ops.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/public/dom0_ops.h Wed Aug 16 17:02:35 2006 +0100
@@ -262,6 +262,18 @@ DEFINE_XEN_GUEST_HANDLE(dom0_sched_id_t)
#define DOM0_SHADOW_CONTROL_OP_CLEAN 11
#define DOM0_SHADOW_CONTROL_OP_PEEK 12
+/* Shadow2 operations */
+#define DOM0_SHADOW2_CONTROL_OP_GET_ALLOCATION 30
+#define DOM0_SHADOW2_CONTROL_OP_SET_ALLOCATION 31
+#define DOM0_SHADOW2_CONTROL_OP_ENABLE 32
+
+/* Mode flags for Shadow2 enable op */
+#define DOM0_SHADOW2_CONTROL_FLAG_ENABLE (1 << 0)
+#define DOM0_SHADOW2_CONTROL_FLAG_REFCOUNT (1 << 1)
+#define DOM0_SHADOW2_CONTROL_FLAG_LOG_DIRTY (1 << 2)
+#define DOM0_SHADOW2_CONTROL_FLAG_TRANSLATE (1 << 3)
+#define DOM0_SHADOW2_CONTROL_FLAG_EXTERNAL (1 << 4)
+
struct dom0_shadow_control_stats {
uint32_t fault_count;
uint32_t dirty_count;
@@ -277,7 +289,9 @@ struct dom0_shadow_control {
uint32_t op;
XEN_GUEST_HANDLE(ulong) dirty_bitmap;
/* IN/OUT variables. */
- uint64_t pages; /* size of buffer, updated with actual size */
+ uint64_t pages; /* size of buffer, updated with actual size */
+ uint32_t mb; /* Shadow2 memory allocation in MB */
+ uint32_t mode; /* Shadow2 mode to enable */
/* OUT variables. */
struct dom0_shadow_control_stats stats;
};
diff -r fda70200da01 -r 0f917d63e960 xen/include/xen/domain_page.h
--- a/xen/include/xen/domain_page.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/xen/domain_page.h Wed Aug 16 17:02:35 2006 +0100
@@ -25,6 +25,13 @@ extern void *map_domain_page(unsigned lo
* currently-executing VCPU via a call to map_domain_pages().
*/
extern void unmap_domain_page(void *va);
+
+/*
+ * Convert a VA (within a page previously mapped in the context of the
+ * currently-executing VCPU via a call to map_domain_pages()) to a machine
+ * address
+ */
+extern paddr_t mapped_domain_page_to_maddr(void *va);
/*
* Similar to the above calls, except the mapping is accessible in all
@@ -98,6 +105,7 @@ domain_mmap_cache_destroy(struct domain_
#define map_domain_page(pfn) maddr_to_virt((pfn)<<PAGE_SHIFT)
#define unmap_domain_page(va) ((void)(va))
+#define mapped_domain_page_to_maddr(va) (virt_to_maddr(va))
#define map_domain_page_global(pfn) maddr_to_virt((pfn)<<PAGE_SHIFT)
#define unmap_domain_page_global(va) ((void)(va))
@@ -112,4 +120,9 @@ struct domain_mmap_cache {
#endif /* !CONFIG_DOMAIN_PAGE */
+#define HERE_I_AM \
+do { \
+ printk("HERE I AM: %s %s %d\n", __func__, __FILE__, __LINE__); \
+} while (0)
+
#endif /* __XEN_DOMAIN_PAGE_H__ */
diff -r fda70200da01 -r 0f917d63e960 xen/include/xen/lib.h
--- a/xen/include/xen/lib.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/xen/lib.h Wed Aug 16 17:02:35 2006 +0100
@@ -18,7 +18,7 @@ extern void __bug(char *file, int line)
#ifndef NDEBUG
#define ASSERT(_p) \
do { \
- if ( !(_p) ) \
+ if ( unlikely(!(_p)) ) \
{ \
printk("Assertion '%s' failed, line %d, file %s\n", #_p , \
__LINE__, __FILE__); \
@@ -41,7 +41,7 @@ void cmdline_parse(char *cmdline);
void cmdline_parse(char *cmdline);
#ifndef NDEBUG
-extern int debugtrace_send_to_console;
+extern void debugtrace_toggle(void);
extern void debugtrace_dump(void);
extern void debugtrace_printk(const char *fmt, ...);
#else
diff -r fda70200da01 -r 0f917d63e960 xen/include/xen/list.h
--- a/xen/include/xen/list.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/xen/list.h Wed Aug 16 17:02:35 2006 +0100
@@ -160,6 +160,16 @@ static __inline__ void list_splice(struc
#define list_for_each_safe(pos, n, head) \
for (pos = (head)->next, n = pos->next; pos != (head); \
pos = n, n = pos->next)
+
+/**
+ * list_for_each_backwards_safe - iterate backwards over a list
safe against removal of list entry
+ * @pos: the &struct list_head to use as a loop counter.
+ * @n: another &struct list_head to use as temporary storage
+ * @head: the head for your list.
+ */
+#define list_for_each_backwards_safe(pos, n, head) \
+ for (pos = (head)->prev, n = pos->prev; pos != (head); \
+ pos = n, n = pos->prev)
/**
* list_for_each_entry - iterate over list of given type
diff -r fda70200da01 -r 0f917d63e960 xen/include/xen/sched.h
--- a/xen/include/xen/sched.h Wed Aug 16 16:16:32 2006 +0100
+++ b/xen/include/xen/sched.h Wed Aug 16 17:02:35 2006 +0100
@@ -376,9 +376,12 @@ extern struct domain *domain_list;
/* VCPU is paused by the hypervisor? */
#define _VCPUF_paused 11
#define VCPUF_paused (1UL<<_VCPUF_paused)
- /* VCPU is blocked awaiting an event to be consumed by Xen. */
+/* VCPU is blocked awaiting an event to be consumed by Xen. */
#define _VCPUF_blocked_in_xen 12
#define VCPUF_blocked_in_xen (1UL<<_VCPUF_blocked_in_xen)
+ /* HVM vcpu thinks CR0.PG == 0 */
+#define _VCPUF_shadow2_translate 13
+#define VCPUF_shadow2_translate (1UL<<_VCPUF_shadow2_translate)
/*
* Per-domain flags (domain_flags).
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/shadow2-common.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/arch/x86/shadow2-common.c Wed Aug 16 17:02:35 2006 +0100
@@ -0,0 +1,3394 @@
+/******************************************************************************
+ * arch/x86/shadow2-common.c
+ *
+ * Shadow2 code that does not need to be multiply compiled.
+ * Parts of this code are Copyright (c) 2006 by XenSource Inc.
+ * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
+ * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#define SHADOW2 1
+
+#include <xen/config.h>
+#include <xen/types.h>
+#include <xen/mm.h>
+#include <xen/trace.h>
+#include <xen/sched.h>
+#include <xen/perfc.h>
+#include <xen/irq.h>
+#include <xen/domain_page.h>
+#include <xen/guest_access.h>
+#include <asm/event.h>
+#include <asm/page.h>
+#include <asm/current.h>
+#include <asm/flushtlb.h>
+#include <asm/shadow2.h>
+#include <asm/shadow2-private.h>
+
+#if SHADOW2_AUDIT
+int shadow2_audit_enable = 0;
+#endif
+
+static void sh2_free_log_dirty_bitmap(struct domain *d);
+
+int _shadow2_mode_refcounts(struct domain *d)
+{
+ return shadow2_mode_refcounts(d);
+}
+
+
+/**************************************************************************/
+/* x86 emulator support for the shadow2 code
+ */
+
+static int
+sh2_x86_emulate_read_std(unsigned long addr,
+ unsigned long *val,
+ unsigned int bytes,
+ struct x86_emulate_ctxt *ctxt)
+{
+ struct vcpu *v = current;
+ if ( hvm_guest(v) )
+ {
+ *val = 0;
+ // XXX -- this is WRONG.
+ // It entirely ignores the permissions in the page tables.
+ // In this case, that is only a user vs supervisor access check.
+ //
+ if ( hvm_copy(val, addr, bytes, HVM_COPY_IN) )
+ {
+#if 0
+ SHADOW2_PRINTK("d=%u v=%u a=%#lx v=%#lx bytes=%u\n",
+ v->domain->domain_id, v->vcpu_id,
+ addr, *val, bytes);
+#endif
+ return X86EMUL_CONTINUE;
+ }
+
+ /* If we got here, there was nothing mapped here, or a bad GFN
+ * was mapped here. This should never happen: we're here because
+ * of a write fault at the end of the instruction we're emulating. */
+ SHADOW2_PRINTK("read failed to va %#lx\n", addr);
+ return X86EMUL_PROPAGATE_FAULT;
+ }
+ else
+ {
+ SHADOW2_PRINTK("this operation is not emulated yet\n");
+ return X86EMUL_UNHANDLEABLE;
+ }
+}
+
+static int
+sh2_x86_emulate_write_std(unsigned long addr,
+ unsigned long val,
+ unsigned int bytes,
+ struct x86_emulate_ctxt *ctxt)
+{
+ struct vcpu *v = current;
+#if 0
+ SHADOW2_PRINTK("d=%u v=%u a=%#lx v=%#lx bytes=%u\n",
+ v->domain->domain_id, v->vcpu_id, addr, val, bytes);
+#endif
+ if ( hvm_guest(v) )
+ {
+ // XXX -- this is WRONG.
+ // It entirely ignores the permissions in the page tables.
+ // In this case, that includes user vs supervisor, and
+ // write access.
+ //
+ if ( hvm_copy(&val, addr, bytes, HVM_COPY_OUT) )
+ return X86EMUL_CONTINUE;
+
+ /* If we got here, there was nothing mapped here, or a bad GFN
+ * was mapped here. This should never happen: we're here because
+ * of a write fault at the end of the instruction we're emulating,
+ * which should be handled by sh2_x86_emulate_write_emulated. */
+ SHADOW2_PRINTK("write failed to va %#lx\n", addr);
+ return X86EMUL_PROPAGATE_FAULT;
+ }
+ else
+ {
+ SHADOW2_PRINTK("this operation is not emulated yet\n");
+ return X86EMUL_UNHANDLEABLE;
+ }
+}
+
+static int
+sh2_x86_emulate_write_emulated(unsigned long addr,
+ unsigned long val,
+ unsigned int bytes,
+ struct x86_emulate_ctxt *ctxt)
+{
+ struct vcpu *v = current;
+#if 0
+ SHADOW2_PRINTK("d=%u v=%u a=%#lx v=%#lx bytes=%u\n",
+ v->domain->domain_id, v->vcpu_id, addr, val, bytes);
+#endif
+ if ( hvm_guest(v) )
+ {
+ return v->arch.shadow2->x86_emulate_write(v, addr, &val, bytes, ctxt);
+ }
+ else
+ {
+ SHADOW2_PRINTK("this operation is not emulated yet\n");
+ return X86EMUL_UNHANDLEABLE;
+ }
+}
+
+static int
+sh2_x86_emulate_cmpxchg_emulated(unsigned long addr,
+ unsigned long old,
+ unsigned long new,
+ unsigned int bytes,
+ struct x86_emulate_ctxt *ctxt)
+{
+ struct vcpu *v = current;
+#if 0
+ SHADOW2_PRINTK("d=%u v=%u a=%#lx o?=%#lx n:=%#lx bytes=%u\n",
+ v->domain->domain_id, v->vcpu_id, addr, old, new, bytes);
+#endif
+ if ( hvm_guest(v) )
+ {
+ return v->arch.shadow2->x86_emulate_cmpxchg(v, addr, old, new,
+ bytes, ctxt);
+ }
+ else
+ {
+ SHADOW2_PRINTK("this operation is not emulated yet\n");
+ return X86EMUL_UNHANDLEABLE;
+ }
+}
+
+static int
+sh2_x86_emulate_cmpxchg8b_emulated(unsigned long addr,
+ unsigned long old_lo,
+ unsigned long old_hi,
+ unsigned long new_lo,
+ unsigned long new_hi,
+ struct x86_emulate_ctxt *ctxt)
+{
+ struct vcpu *v = current;
+#if 0
+ SHADOW2_PRINTK("d=%u v=%u a=%#lx o?=%#lx:%lx n:=%#lx:%lx\n",
+ v->domain->domain_id, v->vcpu_id, addr, old_hi, old_lo,
+ new_hi, new_lo, ctxt);
+#endif
+ if ( hvm_guest(v) )
+ {
+ return v->arch.shadow2->x86_emulate_cmpxchg8b(v, addr, old_lo, old_hi,
+ new_lo, new_hi, ctxt);
+ }
+ else
+ {
+ SHADOW2_PRINTK("this operation is not emulated yet\n");
+ return X86EMUL_UNHANDLEABLE;
+ }
+}
+
+
+struct x86_emulate_ops shadow2_emulator_ops = {
+ .read_std = sh2_x86_emulate_read_std,
+ .write_std = sh2_x86_emulate_write_std,
+ .read_emulated = sh2_x86_emulate_read_std,
+ .write_emulated = sh2_x86_emulate_write_emulated,
+ .cmpxchg_emulated = sh2_x86_emulate_cmpxchg_emulated,
+ .cmpxchg8b_emulated = sh2_x86_emulate_cmpxchg8b_emulated,
+};
+
+
+/**************************************************************************/
+/* Code for "promoting" a guest page to the point where the shadow code is
+ * willing to let it be treated as a guest page table. This generally
+ * involves making sure there are no writable mappings available to the guest
+ * for this page.
+ */
+void shadow2_promote(struct vcpu *v, mfn_t gmfn, u32 type)
+{
+ struct page_info *page = mfn_to_page(gmfn);
+ unsigned long type_info;
+
+ ASSERT(valid_mfn(gmfn));
+
+ /* We should never try to promote a gmfn that has writeable mappings */
+ ASSERT(shadow2_remove_write_access(v, gmfn, 0, 0) == 0);
+
+ // Is the page already shadowed?
+ if ( !test_and_set_bit(_PGC_page_table, &page->count_info) )
+ {
+ // No prior shadow exists...
+
+ // Grab a type-ref. We don't really care if we are racing with another
+ // vcpu or not, or even what kind of type we get; we just want the type
+ // count to be > 0.
+ //
+ do {
+ type_info =
+ page->u.inuse.type_info & (PGT_type_mask | PGT_va_mask);
+ } while ( !get_page_type(page, type_info) );
+
+ // Now that the type ref is non-zero, we can safely use the
+ // shadow2_flags.
+ //
+ page->shadow2_flags = 0;
+ }
+
+ ASSERT(!test_bit(type >> PGC_SH2_type_shift, &page->shadow2_flags));
+ set_bit(type >> PGC_SH2_type_shift, &page->shadow2_flags);
+}
+
+void shadow2_demote(struct vcpu *v, mfn_t gmfn, u32 type)
+{
+ struct page_info *page = mfn_to_page(gmfn);
+
+ ASSERT(test_bit(_PGC_page_table, &page->count_info));
+ ASSERT(test_bit(type >> PGC_SH2_type_shift, &page->shadow2_flags));
+
+ clear_bit(type >> PGC_SH2_type_shift, &page->shadow2_flags);
+
+ if ( (page->shadow2_flags & SH2F_page_type_mask) == 0 )
+ {
+ // release the extra type ref
+ put_page_type(page);
+
+ // clear the is-a-page-table bit.
+ clear_bit(_PGC_page_table, &page->count_info);
+ }
+}
+
+/**************************************************************************/
+/* Validate a pagetable change from the guest and update the shadows.
+ * Returns a bitmask of SHADOW2_SET_* flags. */
+
+static int
+__shadow2_validate_guest_entry(struct vcpu *v, mfn_t gmfn,
+ void *entry, u32 size)
+{
+ int result = 0;
+ struct page_info *page = mfn_to_page(gmfn);
+
+ sh2_mark_dirty(v->domain, gmfn);
+
+ // Determine which types of shadows are affected, and update each.
+ //
+ // Always validate L1s before L2s to prevent another cpu with a linear
+ // mapping of this gmfn from seeing a walk that results from
+ // using the new L2 value and the old L1 value. (It is OK for such a
+ // guest to see a walk that uses the old L2 value with the new L1 value,
+ // as hardware could behave this way if one level of the pagewalk occurs
+ // before the store, and the next level of the pagewalk occurs after the
+ // store.
+ //
+ // Ditto for L2s before L3s, etc.
+ //
+
+ if ( !(page->count_info & PGC_page_table) )
+ return 0; /* Not shadowed at all */
+
+#if CONFIG_PAGING_LEVELS == 2
+ if ( page->shadow2_flags & SH2F_L1_32 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl1e, 2, 2)
+ (v, gmfn, entry, size);
+#else
+ if ( page->shadow2_flags & SH2F_L1_32 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl1e, 3, 2)
+ (v, gmfn, entry, size);
+#endif
+
+#if CONFIG_PAGING_LEVELS == 2
+ if ( page->shadow2_flags & SH2F_L2_32 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2e, 2, 2)
+ (v, gmfn, entry, size);
+#else
+ if ( page->shadow2_flags & SH2F_L2_32 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2e, 3, 2)
+ (v, gmfn, entry, size);
+#endif
+
+#if CONFIG_PAGING_LEVELS >= 3
+ if ( page->shadow2_flags & SH2F_L1_PAE )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl1e, 3, 3)
+ (v, gmfn, entry, size);
+ if ( page->shadow2_flags & SH2F_L2_PAE )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2e, 3, 3)
+ (v, gmfn, entry, size);
+ if ( page->shadow2_flags & SH2F_L2H_PAE )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2he, 3, 3)
+ (v, gmfn, entry, size);
+ if ( page->shadow2_flags & SH2F_L3_PAE )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl3e, 3, 3)
+ (v, gmfn, entry, size);
+#else /* 32-bit non-PAE hypervisor does not support PAE guests */
+ ASSERT((page->shadow2_flags & (SH2F_L3_PAE|SH2F_L2_PAE|SH2F_L1_PAE)) == 0);
+#endif
+
+#if CONFIG_PAGING_LEVELS >= 4
+ if ( page->shadow2_flags & SH2F_L1_64 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl1e, 4, 4)
+ (v, gmfn, entry, size);
+ if ( page->shadow2_flags & SH2F_L2_64 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl2e, 4, 4)
+ (v, gmfn, entry, size);
+ if ( page->shadow2_flags & SH2F_L3_64 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl3e, 4, 4)
+ (v, gmfn, entry, size);
+ if ( page->shadow2_flags & SH2F_L4_64 )
+ result |= SHADOW2_INTERNAL_NAME(sh2_map_and_validate_gl4e, 4, 4)
+ (v, gmfn, entry, size);
+#else /* 32-bit/PAE hypervisor does not support 64-bit guests */
+ ASSERT((page->shadow2_flags
+ & (SH2F_L4_64|SH2F_L3_64|SH2F_L2_64|SH2F_L1_64)) == 0);
+#endif
+
+ return result;
+}
+
+
+int
+shadow2_validate_guest_entry(struct vcpu *v, mfn_t gmfn, void *entry)
+/* This is the entry point from hypercalls. It returns a bitmask of all the
+ * results of shadow_set_l*e() calls, so the caller knows to do TLB flushes. */
+{
+ int rc;
+
+ ASSERT(shadow2_lock_is_acquired(v->domain));
+ rc = __shadow2_validate_guest_entry(v, gmfn, entry, sizeof(l1_pgentry_t));
+ shadow2_audit_tables(v);
+ return rc;
+}
+
+void
+shadow2_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn,
+ void *entry, u32 size)
+/* This is the entry point for emulated writes to pagetables in HVM guests */
+{
+ struct domain *d = v->domain;
+ int rc;
+
+ ASSERT(shadow2_lock_is_acquired(v->domain));
+ rc = __shadow2_validate_guest_entry(v, gmfn, entry, size);
+ if ( rc & SHADOW2_SET_FLUSH )
+ {
+ // Flush everyone except the local processor, which will flush when it
+ // re-enters the HVM guest.
+ //
+ cpumask_t mask = d->domain_dirty_cpumask;
+ cpu_clear(v->processor, mask);
+ flush_tlb_mask(mask);
+ }
+ if ( rc & SHADOW2_SET_ERROR )
+ {
+ /* This page is probably not a pagetable any more: tear it out of the
+ * shadows, along with any tables that reference it */
+ shadow2_remove_all_shadows_and_parents(v, gmfn);
+ }
+ /* We ignore the other bits: since we are about to change CR3 on
+ * VMENTER we don't need to do any extra TLB flushes. */
+}
+
+
+/**************************************************************************/
+/* Memory management for shadow pages. */
+
+/* Meaning of the count_info field in shadow pages
+ * ----------------------------------------------
+ *
+ * A count of all references to this page from other shadow pages and
+ * guest CR3s (a.k.a. v->arch.shadow_table).
+ *
+ * The top bits hold the shadow type and the pinned bit. Top-level
+ * shadows are pinned so that they don't disappear when not in a CR3
+ * somewhere.
+ *
+ * We don't need to use get|put_page for this as the updates are all
+ * protected by the shadow lock. We can't use get|put_page for this
+ * as the size of the count on shadow pages is different from that on
+ * normal guest pages.
+ */
+
+/* Meaning of the type_info field in shadow pages
+ * ----------------------------------------------
+ *
+ * type_info use depends on the shadow type (from count_info)
+ *
+ * PGC_SH2_none : This page is in the shadow2 free pool. type_info holds
+ * the chunk order for our freelist allocator.
+ *
+ * PGC_SH2_l*_shadow : This page is in use as a shadow. type_info
+ * holds the mfn of the guest page being shadowed,
+ *
+ * PGC_SH2_fl1_*_shadow : This page is being used to shatter a superpage.
+ * type_info holds the gfn being shattered.
+ *
+ * PGC_SH2_monitor_table : This page is part of a monitor table.
+ * type_info is not used.
+ */
+
+/* Meaning of the _domain field in shadow pages
+ * --------------------------------------------
+ *
+ * In shadow pages, this field will always have its least significant bit
+ * set. This ensures that all attempts to get_page() will fail (as all
+ * valid pickled domain pointers have a zero for their least significant bit).
+ * Instead, the remaining upper bits are used to record the shadow generation
+ * counter when the shadow was created.
+ */
+
+/* Meaning of the shadow2_flags field
+ * ----------------------------------
+ *
+ * In guest pages that are shadowed, one bit for each kind of shadow they have.
+ *
+ * In shadow pages, will be used for holding a representation of the populated
+ * entries in this shadow (either a min/max, or a bitmap, or ...)
+ *
+ * In monitor-table pages, holds the level of the particular page (to save
+ * spilling the shadow types into an extra bit by having three types of monitor
+ * page).
+ */
+
+/* Meaning of the list_head struct in shadow pages
+ * -----------------------------------------------
+ *
+ * In free shadow pages, this is used to hold the free-lists of chunks.
+ *
+ * In top-level shadow tables, this holds a linked-list of all top-level
+ * shadows (used for recovering memory and destroying shadows).
+ *
+ * In lower-level shadows, this holds the physical address of a higher-level
+ * shadow entry that holds a reference to this shadow (or zero).
+ */
+
+/* Allocating shadow pages
+ * -----------------------
+ *
+ * Most shadow pages are allocated singly, but there are two cases where we
+ * need to allocate multiple pages together.
+ *
+ * 1: Shadowing 32-bit guest tables on PAE or 64-bit shadows.
+ * A 32-bit guest l1 table covers 4MB of virtuial address space,
+ * and needs to be shadowed by two PAE/64-bit l1 tables (covering 2MB
+ * of virtual address space each). Similarly, a 32-bit guest l2 table
+ * (4GB va) needs to be shadowed by four PAE/64-bit l2 tables (1GB va
+ * each). These multi-page shadows are contiguous and aligned;
+ * functions for handling offsets into them are defined in shadow2.c
+ * (shadow_l1_index() etc.)
+ *
+ * 2: Shadowing PAE top-level pages. Each guest page that contains
+ * any PAE top-level pages requires two shadow pages to shadow it.
+ * They contain alternating l3 tables and pae_l3_bookkeeping structs.
+ *
+ * This table shows the allocation behaviour of the different modes:
+ *
+ * Xen paging 32b pae pae 64b 64b 64b
+ * Guest paging 32b 32b pae 32b pae 64b
+ * PV or HVM * HVM * HVM HVM *
+ * Shadow paging 32b pae pae pae pae 64b
+ *
+ * sl1 size 4k 8k 4k 8k 4k 4k
+ * sl2 size 4k 16k 4k 16k 4k 4k
+ * sl3 size - - 8k - 8k 4k
+ * sl4 size - - - - - 4k
+ *
+ * We allocate memory from xen in four-page units and break them down
+ * with a simple buddy allocator. Can't use the xen allocator to handle
+ * this as it only works for contiguous zones, and a domain's shadow
+ * pool is made of fragments.
+ *
+ * In HVM guests, the p2m table is built out of shadow pages, and we provide
+ * a function for the p2m management to steal pages, in max-order chunks, from
+ * the free pool. We don't provide for giving them back, yet.
+ */
+
+/* Figure out the least acceptable quantity of shadow memory.
+ * The minimum memory requirement for always being able to free up a
+ * chunk of memory is very small -- only three max-order chunks per
+ * vcpu to hold the top level shadows and pages with Xen mappings in them.
+ *
+ * But for a guest to be guaranteed to successfully execute a single
+ * instruction, we must be able to map a large number (about thirty) VAs
+ * at the same time, which means that to guarantee progress, we must
+ * allow for more than ninety allocated pages per vcpu. We round that
+ * up to 128 pages, or half a megabyte per vcpu. */
+unsigned int shadow2_min_acceptable_pages(struct domain *d)
+{
+ u32 vcpu_count = 0;
+ struct vcpu *v;
+
+ for_each_vcpu(d, v)
+ vcpu_count++;
+
+ return (vcpu_count * 128);
+}
+
+/* Using the type_info field to store freelist order */
+#define SH2_PFN_ORDER(_p) ((_p)->u.inuse.type_info)
+#define SH2_SET_PFN_ORDER(_p, _o) \
+ do { (_p)->u.inuse.type_info = (_o); } while (0)
+
+
+/* Figure out the order of allocation needed for a given shadow type */
+static inline u32
+shadow_order(u32 shadow_type)
+{
+#if CONFIG_PAGING_LEVELS > 2
+ static const u32 type_to_order[16] = {
+ 0, /* PGC_SH2_none */
+ 1, /* PGC_SH2_l1_32_shadow */
+ 1, /* PGC_SH2_fl1_32_shadow */
+ 2, /* PGC_SH2_l2_32_shadow */
+ 0, /* PGC_SH2_l1_pae_shadow */
+ 0, /* PGC_SH2_fl1_pae_shadow */
+ 0, /* PGC_SH2_l2_pae_shadow */
+ 0, /* PGC_SH2_l2h_pae_shadow */
+ 1, /* PGC_SH2_l3_pae_shadow */
+ 0, /* PGC_SH2_l1_64_shadow */
+ 0, /* PGC_SH2_fl1_64_shadow */
+ 0, /* PGC_SH2_l2_64_shadow */
+ 0, /* PGC_SH2_l3_64_shadow */
+ 0, /* PGC_SH2_l4_64_shadow */
+ 2, /* PGC_SH2_p2m_table */
+ 0 /* PGC_SH2_monitor_table */
+ };
+ u32 type = (shadow_type & PGC_SH2_type_mask) >> PGC_SH2_type_shift;
+ return type_to_order[type];
+#else /* 32-bit Xen only ever shadows 32-bit guests on 32-bit shadows. */
+ return 0;
+#endif
+}
+
+
+/* Do we have a free chunk of at least this order? */
+static inline int chunk_is_available(struct domain *d, int order)
+{
+ int i;
+
+ for ( i = order; i <= SHADOW2_MAX_ORDER; i++ )
+ if ( !list_empty(&d->arch.shadow2_freelists[i]) )
+ return 1;
+ return 0;
+}
+
+/* Dispatcher function: call the per-mode function that will unhook the
+ * non-Xen mappings in this top-level shadow mfn */
+void shadow2_unhook_mappings(struct vcpu *v, mfn_t smfn)
+{
+ struct page_info *pg = mfn_to_page(smfn);
+ switch ( (pg->count_info & PGC_SH2_type_mask) >> PGC_SH2_type_shift )
+ {
+ case PGC_SH2_l2_32_shadow >> PGC_SH2_type_shift:
+#if CONFIG_PAGING_LEVELS == 2
+ SHADOW2_INTERNAL_NAME(sh2_unhook_32b_mappings,2,2)(v,smfn);
+#else
+ SHADOW2_INTERNAL_NAME(sh2_unhook_32b_mappings,3,2)(v,smfn);
+#endif
+ break;
+#if CONFIG_PAGING_LEVELS >= 3
+ case PGC_SH2_l3_pae_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_unhook_pae_mappings,3,3)(v,smfn);
+ break;
+#endif
+#if CONFIG_PAGING_LEVELS >= 4
+ case PGC_SH2_l4_64_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_unhook_64b_mappings,4,4)(v,smfn);
+ break;
+#endif
+ default:
+ SHADOW2_PRINTK("top-level shadow has bad type %08lx\n",
+ (unsigned long)((pg->count_info & PGC_SH2_type_mask)
+ >> PGC_SH2_type_shift));
+ BUG();
+ }
+}
+
+
+/* Make sure there is at least one chunk of the required order available
+ * in the shadow page pool. This must be called before any calls to
+ * shadow2_alloc(). Since this will free existing shadows to make room,
+ * it must be called early enough to avoid freeing shadows that the
+ * caller is currently working on. */
+void shadow2_prealloc(struct domain *d, unsigned int order)
+{
+ /* Need a vpcu for calling unpins; for now, since we don't have
+ * per-vcpu shadows, any will do */
+ struct vcpu *v = d->vcpu[0];
+ struct list_head *l, *t;
+ struct page_info *pg;
+ mfn_t smfn;
+
+ if ( chunk_is_available(d, order) ) return;
+
+ /* Stage one: walk the list of top-level pages, unpinning them */
+ perfc_incrc(shadow2_prealloc_1);
+ list_for_each_backwards_safe(l, t, &d->arch.shadow2_toplevel_shadows)
+ {
+ pg = list_entry(l, struct page_info, list);
+ smfn = page_to_mfn(pg);
+
+#if CONFIG_PAGING_LEVELS >= 3
+ if ( (pg->count_info & PGC_SH2_type_mask) == PGC_SH2_l3_pae_shadow )
+ {
+ /* For PAE, we need to unpin each subshadow on this shadow */
+ SHADOW2_INTERNAL_NAME(sh2_unpin_all_l3_subshadows,3,3)(v, smfn);
+ }
+ else
+#endif /* 32-bit code always takes this branch */
+ {
+ /* Unpin this top-level shadow */
+ sh2_unpin(v, smfn);
+ }
+
+ /* See if that freed up a chunk of appropriate size */
+ if ( chunk_is_available(d, order) ) return;
+ }
+
+ /* Stage two: all shadow pages are in use in hierarchies that are
+ * loaded in cr3 on some vcpu. Walk them, unhooking the non-Xen
+ * mappings. */
+ perfc_incrc(shadow2_prealloc_2);
+ v = current;
+ if ( v->domain != d )
+ v = d->vcpu[0];
+ /* Walk the list from the tail: recently used toplevels have been pulled
+ * to the head */
+ list_for_each_backwards_safe(l, t, &d->arch.shadow2_toplevel_shadows)
+ {
+ pg = list_entry(l, struct page_info, list);
+ smfn = page_to_mfn(pg);
+ shadow2_unhook_mappings(v, smfn);
+
+ /* Need to flush TLB if we've altered our own tables */
+ if ( !shadow2_mode_external(d)
+ && pagetable_get_pfn(current->arch.shadow_table) == mfn_x(smfn) )
+ local_flush_tlb();
+
+ /* See if that freed up a chunk of appropriate size */
+ if ( chunk_is_available(d, order) ) return;
+ }
+
+ /* Nothing more we can do: all remaining shadows are of pages that
+ * hold Xen mappings for some vcpu. This can never happen. */
+ SHADOW2_PRINTK("Can't pre-allocate %i shadow pages!\n"
+ " shadow pages total = %u, free = %u, p2m=%u\n",
+ 1 << order,
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+ BUG();
+}
+
+
+/* Allocate another shadow's worth of (contiguous, aligned) pages,
+ * and fill in the type and backpointer fields of their page_infos.
+ * Never fails to allocate. */
+mfn_t shadow2_alloc(struct domain *d,
+ u32 shadow_type,
+ unsigned long backpointer)
+{
+ struct page_info *pg = NULL;
+ unsigned int order = shadow_order(shadow_type);
+ cpumask_t mask;
+ void *p;
+ int i;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(order <= SHADOW2_MAX_ORDER);
+ ASSERT(shadow_type != PGC_SH2_none);
+ perfc_incrc(shadow2_alloc);
+
+ /* Find smallest order which can satisfy the request. */
+ for ( i = order; i <= SHADOW2_MAX_ORDER; i++ )
+ if ( !list_empty(&d->arch.shadow2_freelists[i]) )
+ {
+ pg = list_entry(d->arch.shadow2_freelists[i].next,
+ struct page_info, list);
+ list_del(&pg->list);
+
+ /* We may have to halve the chunk a number of times. */
+ while ( i != order )
+ {
+ i--;
+ SH2_SET_PFN_ORDER(pg, i);
+ list_add_tail(&pg->list, &d->arch.shadow2_freelists[i]);
+ pg += 1 << i;
+ }
+ d->arch.shadow2_free_pages -= 1 << order;
+
+ /* Init page info fields and clear the pages */
+ for ( i = 0; i < 1<<order ; i++ )
+ {
+ pg[i].u.inuse.type_info = backpointer;
+ pg[i].count_info = shadow_type;
+ pg[i].shadow2_flags = 0;
+ INIT_LIST_HEAD(&pg[i].list);
+ /* Before we overwrite the old contents of this page,
+ * we need to be sure that no TLB holds a pointer to it. */
+ mask = d->domain_dirty_cpumask;
+ tlbflush_filter(mask, pg[i].tlbflush_timestamp);
+ if ( unlikely(!cpus_empty(mask)) )
+ {
+ perfc_incrc(shadow2_alloc_tlbflush);
+ flush_tlb_mask(mask);
+ }
+ /* Now safe to clear the page for reuse */
+ p = sh2_map_domain_page(page_to_mfn(pg+i));
+ ASSERT(p != NULL);
+ clear_page(p);
+ sh2_unmap_domain_page(p);
+ perfc_incr(shadow2_alloc_count);
+ }
+ return page_to_mfn(pg);
+ }
+
+ /* If we get here, we failed to allocate. This should never happen.
+ * It means that we didn't call shadow2_prealloc() correctly before
+ * we allocated. We can't recover by calling prealloc here, because
+ * we might free up higher-level pages that the caller is working on. */
+ SHADOW2_PRINTK("Can't allocate %i shadow pages!\n", 1 << order);
+ BUG();
+}
+
+
+/* Return some shadow pages to the pool. */
+void shadow2_free(struct domain *d, mfn_t smfn)
+{
+ struct page_info *pg = mfn_to_page(smfn);
+ u32 shadow_type;
+ unsigned long order;
+ unsigned long mask;
+ int i;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ perfc_incrc(shadow2_free);
+
+ shadow_type = pg->count_info & PGC_SH2_type_mask;
+ ASSERT(shadow_type != PGC_SH2_none);
+ ASSERT(shadow_type != PGC_SH2_p2m_table);
+ order = shadow_order(shadow_type);
+
+ d->arch.shadow2_free_pages += 1 << order;
+
+ for ( i = 0; i < 1<<order; i++ )
+ {
+ /* Strip out the type: this is now a free shadow page */
+ pg[i].count_info = 0;
+ /* Remember the TLB timestamp so we will know whether to flush
+ * TLBs when we reuse the page. Because the destructors leave the
+ * contents of the pages in place, we can delay TLB flushes until
+ * just before the allocator hands the page out again. */
+ pg[i].tlbflush_timestamp = tlbflush_current_time();
+ perfc_decr(shadow2_alloc_count);
+ }
+
+ /* Merge chunks as far as possible. */
+ while ( order < SHADOW2_MAX_ORDER )
+ {
+ mask = 1 << order;
+ if ( (mfn_x(page_to_mfn(pg)) & mask) ) {
+ /* Merge with predecessor block? */
+ if ( (((pg-mask)->count_info & PGC_SH2_type_mask) != PGT_none)
+ || (SH2_PFN_ORDER(pg-mask) != order) )
+ break;
+ list_del(&(pg-mask)->list);
+ pg -= mask;
+ } else {
+ /* Merge with successor block? */
+ if ( (((pg+mask)->count_info & PGC_SH2_type_mask) != PGT_none)
+ || (SH2_PFN_ORDER(pg+mask) != order) )
+ break;
+ list_del(&(pg+mask)->list);
+ }
+ order++;
+ }
+
+ SH2_SET_PFN_ORDER(pg, order);
+ list_add_tail(&pg->list, &d->arch.shadow2_freelists[order]);
+}
+
+/* Divert some memory from the pool to be used by the p2m mapping.
+ * This action is irreversible: the p2m mapping only ever grows.
+ * That's OK because the p2m table only exists for external domains,
+ * and those domains can't ever turn off shadow mode.
+ * Also, we only ever allocate a max-order chunk, so as to preserve
+ * the invariant that shadow2_prealloc() always works.
+ * Returns 0 iff it can't get a chunk (the caller should then
+ * free up some pages in domheap and call set_sh2_allocation);
+ * returns non-zero on success.
+ */
+static int
+shadow2_alloc_p2m_pages(struct domain *d)
+{
+ struct page_info *pg;
+ u32 i;
+ ASSERT(shadow2_lock_is_acquired(d));
+
+ if ( d->arch.shadow2_total_pages
+ < (shadow2_min_acceptable_pages(d) + (1<<SHADOW2_MAX_ORDER)) )
+ return 0; /* Not enough shadow memory: need to increase it first */
+
+ pg = mfn_to_page(shadow2_alloc(d, PGC_SH2_p2m_table, 0));
+ d->arch.shadow2_p2m_pages += (1<<SHADOW2_MAX_ORDER);
+ d->arch.shadow2_total_pages -= (1<<SHADOW2_MAX_ORDER);
+ for (i = 0; i < (1<<SHADOW2_MAX_ORDER); i++)
+ {
+ /* Unlike shadow pages, mark p2m pages as owned by the domain */
+ page_set_owner(&pg[i], d);
+ list_add_tail(&pg[i].list, &d->arch.shadow2_p2m_freelist);
+ }
+ return 1;
+}
+
+// Returns 0 if no memory is available...
+mfn_t
+shadow2_alloc_p2m_page(struct domain *d)
+{
+ struct list_head *entry;
+ mfn_t mfn;
+ void *p;
+
+ if ( list_empty(&d->arch.shadow2_p2m_freelist) &&
+ !shadow2_alloc_p2m_pages(d) )
+ return _mfn(0);
+ entry = d->arch.shadow2_p2m_freelist.next;
+ list_del(entry);
+ list_add_tail(entry, &d->arch.shadow2_p2m_inuse);
+ mfn = page_to_mfn(list_entry(entry, struct page_info, list));
+ sh2_get_ref(mfn, 0);
+ p = sh2_map_domain_page(mfn);
+ clear_page(p);
+ sh2_unmap_domain_page(p);
+
+ return mfn;
+}
+
+#if CONFIG_PAGING_LEVELS == 3
+static void p2m_install_entry_in_monitors(struct domain *d,
+ l3_pgentry_t *l3e)
+/* Special case, only used for external-mode domains on PAE hosts:
+ * update the mapping of the p2m table. Once again, this is trivial in
+ * other paging modes (one top-level entry points to the top-level p2m,
+ * no maintenance needed), but PAE makes life difficult by needing a
+ * copy the eight l3es of the p2m table in eight l2h slots in the
+ * monitor table. This function makes fresh copies when a p2m l3e
+ * changes. */
+{
+ l2_pgentry_t *ml2e;
+ struct vcpu *v;
+ unsigned int index;
+
+ index = ((unsigned long)l3e & ~PAGE_MASK) / sizeof(l3_pgentry_t);
+ ASSERT(index < MACHPHYS_MBYTES>>1);
+
+ for_each_vcpu(d, v)
+ {
+ if ( pagetable_get_pfn(v->arch.monitor_table) == 0 )
+ continue;
+ ASSERT(shadow2_mode_external(v->domain));
+
+ SHADOW2_DEBUG(P2M, "d=%u v=%u index=%u mfn=%#lx\n",
+ d->domain_id, v->vcpu_id, index, l3e_get_pfn(*l3e));
+
+ if ( v == current ) /* OK to use linear map of monitor_table */
+ ml2e = __linear_l2_table + l2_linear_offset(RO_MPT_VIRT_START);
+ else
+ {
+ l3_pgentry_t *ml3e;
+ ml3e =
sh2_map_domain_page(pagetable_get_mfn(v->arch.monitor_table));
+ ASSERT(l3e_get_flags(ml3e[3]) & _PAGE_PRESENT);
+ ml2e = sh2_map_domain_page(_mfn(l3e_get_pfn(ml3e[3])));
+ ml2e += l2_table_offset(RO_MPT_VIRT_START);
+ sh2_unmap_domain_page(ml3e);
+ }
+ ml2e[index] = l2e_from_pfn(l3e_get_pfn(*l3e), __PAGE_HYPERVISOR);
+ if ( v != current )
+ sh2_unmap_domain_page(ml2e);
+ }
+}
+#endif
+
+// Find the next level's P2M entry, checking for out-of-range gfn's...
+// Returns NULL on error.
+//
+static l1_pgentry_t *
+p2m_find_entry(void *table, unsigned long *gfn_remainder,
+ unsigned long gfn, u32 shift, u32 max)
+{
+ u32 index;
+
+ index = *gfn_remainder >> shift;
+ if ( index >= max )
+ {
+ SHADOW2_DEBUG(P2M, "gfn=0x%lx out of range "
+ "(gfn_remainder=0x%lx shift=%d index=0x%x max=0x%x)\n",
+ gfn, *gfn_remainder, shift, index, max);
+ return NULL;
+ }
+ *gfn_remainder &= (1 << shift) - 1;
+ return (l1_pgentry_t *)table + index;
+}
+
+// Walk one level of the P2M table, allocating a new table if required.
+// Returns 0 on error.
+//
+static int
+p2m_next_level(struct domain *d, mfn_t *table_mfn, void **table,
+ unsigned long *gfn_remainder, unsigned long gfn, u32 shift,
+ u32 max, unsigned long type)
+{
+ l1_pgentry_t *p2m_entry;
+ void *next;
+
+ if ( !(p2m_entry = p2m_find_entry(*table, gfn_remainder, gfn,
+ shift, max)) )
+ return 0;
+
+ if ( !(l1e_get_flags(*p2m_entry) & _PAGE_PRESENT) )
+ {
+ mfn_t mfn = shadow2_alloc_p2m_page(d);
+ if ( mfn_x(mfn) == 0 )
+ return 0;
+ *p2m_entry = l1e_from_pfn(mfn_x(mfn), __PAGE_HYPERVISOR|_PAGE_USER);
+ mfn_to_page(mfn)->u.inuse.type_info = type | 1 | PGT_validated;
+ mfn_to_page(mfn)->count_info = 1;
+#if CONFIG_PAGING_LEVELS == 3
+ if (type == PGT_l2_page_table)
+ {
+ /* We have written to the p2m l3: need to sync the per-vcpu
+ * copies of it in the monitor tables */
+ p2m_install_entry_in_monitors(d, (l3_pgentry_t *)p2m_entry);
+ }
+#endif
+ /* The P2M can be shadowed: keep the shadows synced */
+ if ( d->vcpu[0] )
+ (void)__shadow2_validate_guest_entry(d->vcpu[0], *table_mfn,
+ p2m_entry, sizeof *p2m_entry);
+ }
+ *table_mfn = _mfn(l1e_get_pfn(*p2m_entry));
+ next = sh2_map_domain_page(*table_mfn);
+ sh2_unmap_domain_page(*table);
+ *table = next;
+
+ return 1;
+}
+
+// Returns 0 on error (out of memory)
+int
+shadow2_set_p2m_entry(struct domain *d, unsigned long gfn, mfn_t mfn)
+{
+ // XXX -- this might be able to be faster iff current->domain == d
+ mfn_t table_mfn = pagetable_get_mfn(d->arch.phys_table);
+ void *table = sh2_map_domain_page(table_mfn);
+ unsigned long gfn_remainder = gfn;
+ l1_pgentry_t *p2m_entry;
+
+#if CONFIG_PAGING_LEVELS >= 4
+ if ( !p2m_next_level(d, &table_mfn, &table, &gfn_remainder, gfn,
+ L4_PAGETABLE_SHIFT - PAGE_SHIFT,
+ L4_PAGETABLE_ENTRIES, PGT_l3_page_table) )
+ return 0;
+#endif
+#if CONFIG_PAGING_LEVELS >= 3
+ // When using PAE Xen, we only allow 33 bits of pseudo-physical
+ // address in translated guests (i.e. 8 GBytes). This restriction
+ // comes from wanting to map the P2M table into the 16MB RO_MPT hole
+ // in Xen's address space for translated PV guests.
+ //
+ if ( !p2m_next_level(d, &table_mfn, &table, &gfn_remainder, gfn,
+ L3_PAGETABLE_SHIFT - PAGE_SHIFT,
+ (CONFIG_PAGING_LEVELS == 3
+ ? 8
+ : L3_PAGETABLE_ENTRIES),
+ PGT_l2_page_table) )
+ return 0;
+#endif
+ if ( !p2m_next_level(d, &table_mfn, &table, &gfn_remainder, gfn,
+ L2_PAGETABLE_SHIFT - PAGE_SHIFT,
+ L2_PAGETABLE_ENTRIES, PGT_l1_page_table) )
+ return 0;
+
+ p2m_entry = p2m_find_entry(table, &gfn_remainder, gfn,
+ 0, L1_PAGETABLE_ENTRIES);
+ ASSERT(p2m_entry);
+ if ( valid_mfn(mfn) )
+ *p2m_entry = l1e_from_pfn(mfn_x(mfn), __PAGE_HYPERVISOR|_PAGE_USER);
+ else
+ *p2m_entry = l1e_empty();
+
+ /* The P2M can be shadowed: keep the shadows synced */
+ (void) __shadow2_validate_guest_entry(d->vcpu[0], table_mfn,
+ p2m_entry, sizeof *p2m_entry);
+
+ sh2_unmap_domain_page(table);
+
+ return 1;
+}
+
+// Allocate a new p2m table for a domain.
+//
+// The structure of the p2m table is that of a pagetable for xen (i.e. it is
+// controlled by CONFIG_PAGING_LEVELS).
+//
+// Returns 0 if p2m table could not be initialized
+//
+static int
+shadow2_alloc_p2m_table(struct domain *d)
+{
+ mfn_t p2m_top;
+ struct list_head *entry;
+ unsigned int page_count = 0;
+
+ SHADOW2_PRINTK("allocating p2m table\n");
+ ASSERT(pagetable_get_pfn(d->arch.phys_table) == 0);
+
+ p2m_top = shadow2_alloc_p2m_page(d);
+ mfn_to_page(p2m_top)->count_info = 1;
+ mfn_to_page(p2m_top)->u.inuse.type_info =
+#if CONFIG_PAGING_LEVELS == 4
+ PGT_l4_page_table
+#elif CONFIG_PAGING_LEVELS == 3
+ PGT_l3_page_table
+#elif CONFIG_PAGING_LEVELS == 2
+ PGT_l2_page_table
+#endif
+ | 1 | PGT_validated;
+
+ if ( mfn_x(p2m_top) == 0 )
+ return 0;
+
+ d->arch.phys_table = pagetable_from_mfn(p2m_top);
+
+ SHADOW2_PRINTK("populating p2m table\n");
+
+ for ( entry = d->page_list.next;
+ entry != &d->page_list;
+ entry = entry->next )
+ {
+ struct page_info *page = list_entry(entry, struct page_info, list);
+ mfn_t mfn = page_to_mfn(page);
+ unsigned long gfn = get_gpfn_from_mfn(mfn_x(mfn));
+ page_count++;
+ if (
+#ifdef __x86_64__
+ (gfn != 0x5555555555555555L)
+#else
+ (gfn != 0x55555555L)
+#endif
+ && gfn != INVALID_M2P_ENTRY
+ && !shadow2_set_p2m_entry(d, gfn, mfn) )
+ {
+ SHADOW2_PRINTK("failed to initialize p2m table, gfn=%05lx, mfn=%"
SH2_PRI_mfn "\n",
+ gfn, mfn_x(mfn));
+ return 0;
+ }
+ }
+
+ SHADOW2_PRINTK("p2m table initialised (%u pages)\n", page_count);
+ return 1;
+}
+
+mfn_t
+sh2_gfn_to_mfn_foreign(struct domain *d, unsigned long gpfn)
+/* Read another domain's p2m entries */
+{
+ mfn_t mfn;
+ unsigned long addr = gpfn << PAGE_SHIFT;
+ l2_pgentry_t *l2e;
+ l1_pgentry_t *l1e;
+
+ ASSERT(shadow2_mode_translate(d));
+ mfn = pagetable_get_mfn(d->arch.phys_table);
+
+
+#if CONFIG_PAGING_LEVELS > 2
+ if ( gpfn > (RO_MPT_VIRT_END - RO_MPT_VIRT_START) / sizeof(l1_pgentry_t) )
+ /* This pfn is higher than the p2m map can hold */
+ return _mfn(INVALID_MFN);
+#endif
+
+
+#if CONFIG_PAGING_LEVELS >= 4
+ {
+ l4_pgentry_t *l4e = sh2_map_domain_page(mfn);
+ l4e += l4_table_offset(addr);
+ if ( (l4e_get_flags(*l4e) & _PAGE_PRESENT) == 0 )
+ {
+ sh2_unmap_domain_page(l4e);
+ return _mfn(INVALID_MFN);
+ }
+ mfn = _mfn(l4e_get_pfn(*l4e));
+ sh2_unmap_domain_page(l4e);
+ }
+#endif
+#if CONFIG_PAGING_LEVELS >= 3
+ {
+ l3_pgentry_t *l3e = sh2_map_domain_page(mfn);
+ l3e += l3_table_offset(addr);
+ if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
+ {
+ sh2_unmap_domain_page(l3e);
+ return _mfn(INVALID_MFN);
+ }
+ mfn = _mfn(l3e_get_pfn(*l3e));
+ sh2_unmap_domain_page(l3e);
+ }
+#endif
+
+ l2e = sh2_map_domain_page(mfn);
+ l2e += l2_table_offset(addr);
+ if ( (l2e_get_flags(*l2e) & _PAGE_PRESENT) == 0 )
+ {
+ sh2_unmap_domain_page(l2e);
+ return _mfn(INVALID_MFN);
+ }
+ mfn = _mfn(l2e_get_pfn(*l2e));
+ sh2_unmap_domain_page(l2e);
+
+ l1e = sh2_map_domain_page(mfn);
+ l1e += l1_table_offset(addr);
+ if ( (l1e_get_flags(*l1e) & _PAGE_PRESENT) == 0 )
+ {
+ sh2_unmap_domain_page(l1e);
+ return _mfn(INVALID_MFN);
+ }
+ mfn = _mfn(l1e_get_pfn(*l1e));
+ sh2_unmap_domain_page(l1e);
+
+ return mfn;
+}
+
+unsigned long
+shadow2_gfn_to_mfn_foreign(unsigned long gpfn)
+{
+ return mfn_x(sh2_gfn_to_mfn_foreign(current->domain, gpfn));
+}
+
+
+static void shadow2_p2m_teardown(struct domain *d)
+/* Return all the p2m pages to Xen.
+ * We know we don't have any extra mappings to these pages */
+{
+ struct list_head *entry, *n;
+ struct page_info *pg;
+
+ d->arch.phys_table = pagetable_null();
+
+ list_for_each_safe(entry, n, &d->arch.shadow2_p2m_inuse)
+ {
+ pg = list_entry(entry, struct page_info, list);
+ list_del(entry);
+ /* Should have just the one ref we gave it in alloc_p2m_page() */
+ if ( (pg->count_info & PGC_SH2_count_mask) != 1 )
+ {
+ SHADOW2_PRINTK("Odd p2m page count c=%#x t=%"PRtype_info"\n",
+ pg->count_info, pg->u.inuse.type_info);
+ }
+ ASSERT(page_get_owner(pg) == d);
+ /* Free should not decrement domain's total allocation, since
+ * these pages were allocated without an owner. */
+ page_set_owner(pg, NULL);
+ free_domheap_pages(pg, 0);
+ d->arch.shadow2_p2m_pages--;
+ perfc_decr(shadow2_alloc_count);
+ }
+ list_for_each_safe(entry, n, &d->arch.shadow2_p2m_freelist)
+ {
+ list_del(entry);
+ pg = list_entry(entry, struct page_info, list);
+ ASSERT(page_get_owner(pg) == d);
+ /* Free should not decrement domain's total allocation. */
+ page_set_owner(pg, NULL);
+ free_domheap_pages(pg, 0);
+ d->arch.shadow2_p2m_pages--;
+ perfc_decr(shadow2_alloc_count);
+ }
+ ASSERT(d->arch.shadow2_p2m_pages == 0);
+}
+
+/* Set the pool of shadow pages to the required number of pages.
+ * Input will be rounded up to at least shadow2_min_acceptable_pages(),
+ * plus space for the p2m table.
+ * Returns 0 for success, non-zero for failure. */
+static unsigned int set_sh2_allocation(struct domain *d,
+ unsigned int pages,
+ int *preempted)
+{
+ struct page_info *pg;
+ unsigned int lower_bound;
+ int j;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+
+ /* Don't allocate less than the minimum acceptable, plus one page per
+ * megabyte of RAM (for the p2m table) */
+ lower_bound = shadow2_min_acceptable_pages(d) + (d->tot_pages / 256);
+ if ( pages > 0 && pages < lower_bound )
+ pages = lower_bound;
+ /* Round up to largest block size */
+ pages = (pages + ((1<<SHADOW2_MAX_ORDER)-1)) & ~((1<<SHADOW2_MAX_ORDER)-1);
+
+ SHADOW2_PRINTK("current %i target %i\n",
+ d->arch.shadow2_total_pages, pages);
+
+ while ( d->arch.shadow2_total_pages != pages )
+ {
+ if ( d->arch.shadow2_total_pages < pages )
+ {
+ /* Need to allocate more memory from domheap */
+ pg = alloc_domheap_pages(NULL, SHADOW2_MAX_ORDER, 0);
+ if ( pg == NULL )
+ {
+ SHADOW2_PRINTK("failed to allocate shadow pages.\n");
+ return -ENOMEM;
+ }
+ d->arch.shadow2_free_pages += 1<<SHADOW2_MAX_ORDER;
+ d->arch.shadow2_total_pages += 1<<SHADOW2_MAX_ORDER;
+ for ( j = 0; j < 1<<SHADOW2_MAX_ORDER; j++ )
+ {
+ pg[j].u.inuse.type_info = 0; /* Free page */
+ pg[j].tlbflush_timestamp = 0; /* Not in any TLB */
+ }
+ SH2_SET_PFN_ORDER(pg, SHADOW2_MAX_ORDER);
+ list_add_tail(&pg->list,
+ &d->arch.shadow2_freelists[SHADOW2_MAX_ORDER]);
+ }
+ else if ( d->arch.shadow2_total_pages > pages )
+ {
+ /* Need to return memory to domheap */
+ shadow2_prealloc(d, SHADOW2_MAX_ORDER);
+ ASSERT(!list_empty(&d->arch.shadow2_freelists[SHADOW2_MAX_ORDER]));
+ pg = list_entry(d->arch.shadow2_freelists[SHADOW2_MAX_ORDER].next,
+ struct page_info, list);
+ list_del(&pg->list);
+ d->arch.shadow2_free_pages -= 1<<SHADOW2_MAX_ORDER;
+ d->arch.shadow2_total_pages -= 1<<SHADOW2_MAX_ORDER;
+ free_domheap_pages(pg, SHADOW2_MAX_ORDER);
+ }
+
+ /* Check to see if we need to yield and try again */
+ if ( preempted && hypercall_preempt_check() )
+ {
+ *preempted = 1;
+ return 0;
+ }
+ }
+
+ return 0;
+}
+
+unsigned int shadow2_set_allocation(struct domain *d,
+ unsigned int megabytes,
+ int *preempted)
+/* Hypercall interface to set the shadow memory allocation */
+{
+ unsigned int rv;
+ shadow2_lock(d);
+ rv = set_sh2_allocation(d, megabytes << (20 - PAGE_SHIFT), preempted);
+ SHADOW2_PRINTK("dom %u allocation now %u pages (%u MB)\n",
+ d->domain_id,
+ d->arch.shadow2_total_pages,
+ shadow2_get_allocation(d));
+ shadow2_unlock(d);
+ return rv;
+}
+
+/**************************************************************************/
+/* Hash table for storing the guest->shadow mappings */
+
+/* Hash function that takes a gfn or mfn, plus another byte of type info */
+typedef u32 key_t;
+static inline key_t sh2_hash(unsigned long n, u8 t)
+{
+ unsigned char *p = (unsigned char *)&n;
+ key_t k = t;
+ int i;
+ for ( i = 0; i < sizeof(n) ; i++ ) k = (u32)p[i] + (k<<6) + (k<<16) - k;
+ return k;
+}
+
+#if SHADOW2_AUDIT & (SHADOW2_AUDIT_HASH|SHADOW2_AUDIT_HASH_FULL)
+
+/* Before we get to the mechanism, define a pair of audit functions
+ * that sanity-check the contents of the hash table. */
+static void sh2_hash_audit_bucket(struct domain *d, int bucket)
+/* Audit one bucket of the hash table */
+{
+ struct shadow2_hash_entry *e, *x;
+ struct page_info *pg;
+
+ if ( !(SHADOW2_AUDIT_ENABLE) )
+ return;
+
+ e = &d->arch.shadow2_hash_table[bucket];
+ if ( e->t == 0 ) return; /* Bucket is empty */
+ while ( e )
+ {
+ /* Empty link? */
+ BUG_ON( e->t == 0 );
+ /* Bogus type? */
+ BUG_ON( e->t > (PGC_SH2_max_shadow >> PGC_SH2_type_shift) );
+ /* Wrong bucket? */
+ BUG_ON( sh2_hash(e->n, e->t) % SHADOW2_HASH_BUCKETS != bucket );
+ /* Duplicate entry? */
+ for ( x = e->next; x; x = x->next )
+ BUG_ON( x->n == e->n && x->t == e->t );
+ /* Bogus MFN? */
+ BUG_ON( !valid_mfn(e->smfn) );
+ pg = mfn_to_page(e->smfn);
+ /* Not a shadow? */
+ BUG_ON( page_get_owner(pg) != 0 );
+ /* Wrong kind of shadow? */
+ BUG_ON( (pg->count_info & PGC_SH2_type_mask) >> PGC_SH2_type_shift
+ != e->t );
+ /* Bad backlink? */
+ BUG_ON( pg->u.inuse.type_info != e->n );
+ if ( e->t != (PGC_SH2_fl1_32_shadow >> PGC_SH2_type_shift)
+ && e->t != (PGC_SH2_fl1_pae_shadow >> PGC_SH2_type_shift)
+ && e->t != (PGC_SH2_fl1_64_shadow >> PGC_SH2_type_shift) )
+ {
+ /* Bad shadow flags on guest page? */
+ BUG_ON( !(mfn_to_page(_mfn(e->n))->shadow2_flags & (1<<e->t)) );
+ }
+ /* That entry was OK; on we go */
+ e = e->next;
+ }
+}
+
+#else
+#define sh2_hash_audit_bucket(_d, _b)
+#endif /* Hashtable bucket audit */
+
+
+#if SHADOW2_AUDIT & SHADOW2_AUDIT_HASH_FULL
+
+static void sh2_hash_audit(struct domain *d)
+/* Full audit: audit every bucket in the table */
+{
+ int i;
+
+ if ( !(SHADOW2_AUDIT_ENABLE) )
+ return;
+
+ for ( i = 0; i < SHADOW2_HASH_BUCKETS; i++ )
+ {
+ sh2_hash_audit_bucket(d, i);
+ }
+}
+
+#else
+#define sh2_hash_audit(_d)
+#endif /* Hashtable bucket audit */
+
+/* Memory management interface for bucket allocation.
+ * These ought to come out of shadow memory, but at least on 32-bit
+ * machines we are forced to allocate them from xenheap so that we can
+ * address them. */
+static struct shadow2_hash_entry *sh2_alloc_hash_entry(struct domain *d)
+{
+ struct shadow2_hash_entry *extra, *x;
+ int i;
+
+ /* We need to allocate a new node. Ensure the free list is not empty.
+ * Allocate new entries in units the same size as the original table. */
+ if ( unlikely(d->arch.shadow2_hash_freelist == NULL) )
+ {
+ size_t sz = sizeof(void *) + (SHADOW2_HASH_BUCKETS * sizeof(*x));
+ extra = xmalloc_bytes(sz);
+
+ if ( extra == NULL )
+ {
+ /* No memory left! */
+ SHADOW2_ERROR("xmalloc() failed when allocating hash buckets.\n");
+ domain_crash_synchronous();
+ }
+ memset(extra, 0, sz);
+
+ /* Record the allocation block so it can be correctly freed later. */
+ *((struct shadow2_hash_entry **)&extra[SHADOW2_HASH_BUCKETS]) =
+ d->arch.shadow2_hash_allocations;
+ d->arch.shadow2_hash_allocations = &extra[0];
+
+ /* Thread a free chain through the newly-allocated nodes. */
+ for ( i = 0; i < (SHADOW2_HASH_BUCKETS - 1); i++ )
+ extra[i].next = &extra[i+1];
+ extra[i].next = NULL;
+
+ /* Add the new nodes to the free list. */
+ d->arch.shadow2_hash_freelist = &extra[0];
+ }
+
+ /* Allocate a new node from the free list. */
+ x = d->arch.shadow2_hash_freelist;
+ d->arch.shadow2_hash_freelist = x->next;
+ return x;
+}
+
+static void sh2_free_hash_entry(struct domain *d, struct shadow2_hash_entry *e)
+{
+ /* Mark the bucket as empty and return it to the free list */
+ e->t = 0;
+ e->next = d->arch.shadow2_hash_freelist;
+ d->arch.shadow2_hash_freelist = e;
+}
+
+
+/* Allocate and initialise the table itself.
+ * Returns 0 for success, 1 for error. */
+static int shadow2_hash_alloc(struct domain *d)
+{
+ struct shadow2_hash_entry *table;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(!d->arch.shadow2_hash_table);
+
+ table = xmalloc_array(struct shadow2_hash_entry, SHADOW2_HASH_BUCKETS);
+ if ( !table ) return 1;
+ memset(table, 0,
+ SHADOW2_HASH_BUCKETS * sizeof (struct shadow2_hash_entry));
+ d->arch.shadow2_hash_table = table;
+ return 0;
+}
+
+/* Tear down the hash table and return all memory to Xen.
+ * This function does not care whether the table is populated. */
+static void shadow2_hash_teardown(struct domain *d)
+{
+ struct shadow2_hash_entry *a, *n;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(d->arch.shadow2_hash_table);
+
+ /* Return the table itself */
+ xfree(d->arch.shadow2_hash_table);
+ d->arch.shadow2_hash_table = NULL;
+
+ /* Return any extra allocations */
+ a = d->arch.shadow2_hash_allocations;
+ while ( a )
+ {
+ /* We stored a linked-list pointer at the end of each allocation */
+ n = *((struct shadow2_hash_entry **)(&a[SHADOW2_HASH_BUCKETS]));
+ xfree(a);
+ a = n;
+ }
+ d->arch.shadow2_hash_allocations = NULL;
+ d->arch.shadow2_hash_freelist = NULL;
+}
+
+
+mfn_t shadow2_hash_lookup(struct vcpu *v, unsigned long n, u8 t)
+/* Find an entry in the hash table. Returns the MFN of the shadow,
+ * or INVALID_MFN if it doesn't exist */
+{
+ struct domain *d = v->domain;
+ struct shadow2_hash_entry *p, *x, *head;
+ key_t key;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(d->arch.shadow2_hash_table);
+ ASSERT(t);
+
+ sh2_hash_audit(d);
+
+ perfc_incrc(shadow2_hash_lookups);
+ key = sh2_hash(n, t);
+
+ x = head = &d->arch.shadow2_hash_table[key % SHADOW2_HASH_BUCKETS];
+ p = NULL;
+
+ sh2_hash_audit_bucket(d, key % SHADOW2_HASH_BUCKETS);
+
+ do
+ {
+ ASSERT(x->t || ((x == head) && (x->next == NULL)));
+
+ if ( x->n == n && x->t == t )
+ {
+ /* Pull-to-front if 'x' isn't already the head item */
+ if ( unlikely(x != head) )
+ {
+ if ( unlikely(d->arch.shadow2_hash_walking != 0) )
+ /* Can't reorder: someone is walking the hash chains */
+ return x->smfn;
+ else
+ {
+ /* Delete 'x' from list and reinsert after head. */
+ p->next = x->next;
+ x->next = head->next;
+ head->next = x;
+
+ /* Swap 'x' contents with head contents. */
+ SWAP(head->n, x->n);
+ SWAP(head->t, x->t);
+ SWAP(head->smfn, x->smfn);
+ }
+ }
+ else
+ {
+ perfc_incrc(shadow2_hash_lookup_head);
+ }
+ return head->smfn;
+ }
+
+ p = x;
+ x = x->next;
+ }
+ while ( x != NULL );
+
+ perfc_incrc(shadow2_hash_lookup_miss);
+ return _mfn(INVALID_MFN);
+}
+
+void shadow2_hash_insert(struct vcpu *v, unsigned long n, u8 t, mfn_t smfn)
+/* Put a mapping (n,t)->smfn into the hash table */
+{
+ struct domain *d = v->domain;
+ struct shadow2_hash_entry *x, *head;
+ key_t key;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(d->arch.shadow2_hash_table);
+ ASSERT(t);
+
+ sh2_hash_audit(d);
+
+ perfc_incrc(shadow2_hash_inserts);
+ key = sh2_hash(n, t);
+
+ head = &d->arch.shadow2_hash_table[key % SHADOW2_HASH_BUCKETS];
+
+ sh2_hash_audit_bucket(d, key % SHADOW2_HASH_BUCKETS);
+
+ /* If the bucket is empty then insert the new page as the head item. */
+ if ( head->t == 0 )
+ {
+ head->n = n;
+ head->t = t;
+ head->smfn = smfn;
+ ASSERT(head->next == NULL);
+ }
+ else
+ {
+ /* Insert a new entry directly after the head item. */
+ x = sh2_alloc_hash_entry(d);
+ x->n = n;
+ x->t = t;
+ x->smfn = smfn;
+ x->next = head->next;
+ head->next = x;
+ }
+
+ sh2_hash_audit_bucket(d, key % SHADOW2_HASH_BUCKETS);
+}
+
+void shadow2_hash_delete(struct vcpu *v, unsigned long n, u8 t, mfn_t smfn)
+/* Excise the mapping (n,t)->smfn from the hash table */
+{
+ struct domain *d = v->domain;
+ struct shadow2_hash_entry *p, *x, *head;
+ key_t key;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(d->arch.shadow2_hash_table);
+ ASSERT(t);
+
+ sh2_hash_audit(d);
+
+ perfc_incrc(shadow2_hash_deletes);
+ key = sh2_hash(n, t);
+
+ head = &d->arch.shadow2_hash_table[key % SHADOW2_HASH_BUCKETS];
+
+ sh2_hash_audit_bucket(d, key % SHADOW2_HASH_BUCKETS);
+
+ /* Match on head item? */
+ if ( head->n == n && head->t == t )
+ {
+ if ( (x = head->next) != NULL )
+ {
+ /* Overwrite head with contents of following node. */
+ head->n = x->n;
+ head->t = x->t;
+ head->smfn = x->smfn;
+
+ /* Delete following node. */
+ head->next = x->next;
+ sh2_free_hash_entry(d, x);
+ }
+ else
+ {
+ /* This bucket is now empty. Initialise the head node. */
+ head->t = 0;
+ }
+ }
+ else
+ {
+ /* Not at the head; need to walk the chain */
+ p = head;
+ x = head->next;
+
+ while(1)
+ {
+ ASSERT(x); /* We can't have hit the end, since our target is
+ * still in the chain somehwere... */
+ if ( x->n == n && x->t == t )
+ {
+ /* Delete matching node. */
+ p->next = x->next;
+ sh2_free_hash_entry(d, x);
+ break;
+ }
+ p = x;
+ x = x->next;
+ }
+ }
+
+ sh2_hash_audit_bucket(d, key % SHADOW2_HASH_BUCKETS);
+}
+
+typedef int (*hash_callback_t)(struct vcpu *v, mfn_t smfn, mfn_t other_mfn);
+
+static void hash_foreach(struct vcpu *v,
+ unsigned int callback_mask,
+ hash_callback_t callbacks[],
+ mfn_t callback_mfn)
+/* Walk the hash table looking at the types of the entries and
+ * calling the appropriate callback function for each entry.
+ * The mask determines which shadow types we call back for, and the array
+ * of callbacks tells us which function to call.
+ * Any callback may return non-zero to let us skip the rest of the scan.
+ *
+ * WARNING: Callbacks MUST NOT add or remove hash entries unless they
+ * then return non-zero to terminate the scan. */
+{
+ int i, done = 0;
+ struct domain *d = v->domain;
+ struct shadow2_hash_entry *x;
+
+ /* Say we're here, to stop hash-lookups reordering the chains */
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(d->arch.shadow2_hash_walking == 0);
+ d->arch.shadow2_hash_walking = 1;
+
+ callback_mask &= ~1; /* Never attempt to call back on empty buckets */
+ for ( i = 0; i < SHADOW2_HASH_BUCKETS; i++ )
+ {
+ /* WARNING: This is not safe against changes to the hash table.
+ * The callback *must* return non-zero if it has inserted or
+ * deleted anything from the hash (lookups are OK, though). */
+ for ( x = &d->arch.shadow2_hash_table[i]; x; x = x->next )
+ {
+ if ( callback_mask & (1 << x->t) )
+ {
+ ASSERT(x->t <= 15);
+ ASSERT(callbacks[x->t] != NULL);
+ if ( (done = callbacks[x->t](v, x->smfn, callback_mfn)) != 0 )
+ break;
+ }
+ }
+ if ( done ) break;
+ }
+ d->arch.shadow2_hash_walking = 0;
+}
+
+
+/**************************************************************************/
+/* Destroy a shadow page: simple dispatcher to call the per-type destructor
+ * which will decrement refcounts appropriately and return memory to the
+ * free pool. */
+
+void sh2_destroy_shadow(struct vcpu *v, mfn_t smfn)
+{
+ struct page_info *pg = mfn_to_page(smfn);
+ u32 t = pg->count_info & PGC_SH2_type_mask;
+
+
+ SHADOW2_PRINTK("smfn=%#lx\n", mfn_x(smfn));
+
+ /* Double-check, if we can, that the shadowed page belongs to this
+ * domain, (by following the back-pointer). */
+ ASSERT(t == PGC_SH2_fl1_32_shadow ||
+ t == PGC_SH2_fl1_pae_shadow ||
+ t == PGC_SH2_fl1_64_shadow ||
+ t == PGC_SH2_monitor_table ||
+ (page_get_owner(mfn_to_page(_mfn(pg->u.inuse.type_info)))
+ == v->domain));
+
+ /* The down-shifts here are so that the switch statement is on nice
+ * small numbers that the compiler will enjoy */
+ switch ( t >> PGC_SH2_type_shift )
+ {
+#if CONFIG_PAGING_LEVELS == 2
+ case PGC_SH2_l1_32_shadow >> PGC_SH2_type_shift:
+ case PGC_SH2_fl1_32_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l1_shadow, 2, 2)(v, smfn);
+ break;
+ case PGC_SH2_l2_32_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l2_shadow, 2, 2)(v, smfn);
+ break;
+#else /* PAE or 64bit */
+ case PGC_SH2_l1_32_shadow >> PGC_SH2_type_shift:
+ case PGC_SH2_fl1_32_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l1_shadow, 3, 2)(v, smfn);
+ break;
+ case PGC_SH2_l2_32_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l2_shadow, 3, 2)(v, smfn);
+ break;
+#endif
+
+#if CONFIG_PAGING_LEVELS >= 3
+ case PGC_SH2_l1_pae_shadow >> PGC_SH2_type_shift:
+ case PGC_SH2_fl1_pae_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l1_shadow, 3, 3)(v, smfn);
+ break;
+ case PGC_SH2_l2_pae_shadow >> PGC_SH2_type_shift:
+ case PGC_SH2_l2h_pae_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l2_shadow, 3, 3)(v, smfn);
+ break;
+ case PGC_SH2_l3_pae_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l3_shadow, 3, 3)(v, smfn);
+ break;
+#endif
+
+#if CONFIG_PAGING_LEVELS >= 4
+ case PGC_SH2_l1_64_shadow >> PGC_SH2_type_shift:
+ case PGC_SH2_fl1_64_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l1_shadow, 4, 4)(v, smfn);
+ break;
+ case PGC_SH2_l2_64_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l2_shadow, 4, 4)(v, smfn);
+ break;
+ case PGC_SH2_l3_64_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l3_shadow, 4, 4)(v, smfn);
+ break;
+ case PGC_SH2_l4_64_shadow >> PGC_SH2_type_shift:
+ SHADOW2_INTERNAL_NAME(sh2_destroy_l4_shadow, 4, 4)(v, smfn);
+ break;
+#endif
+ default:
+ SHADOW2_PRINTK("tried to destroy shadow of bad type %08lx\n",
+ (unsigned long)t);
+ BUG();
+ }
+}
+
+/**************************************************************************/
+/* Remove all writeable mappings of a guest frame from the shadow tables
+ * Returns non-zero if we need to flush TLBs.
+ * level and fault_addr desribe how we found this to be a pagetable;
+ * level==0 means we have some other reason for revoking write access.*/
+
+int shadow2_remove_write_access(struct vcpu *v, mfn_t gmfn,
+ unsigned int level,
+ unsigned long fault_addr)
+{
+ /* Dispatch table for getting per-type functions */
+ static hash_callback_t callbacks[16] = {
+ NULL, /* none */
+#if CONFIG_PAGING_LEVELS == 2
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,2,2), /* l1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,2,2), /* fl1_32 */
+#else
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,3,2), /* l1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,3,2), /* fl1_32 */
+#endif
+ NULL, /* l2_32 */
+#if CONFIG_PAGING_LEVELS >= 3
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,3,3), /* l1_pae */
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,3,3), /* fl1_pae */
+#else
+ NULL, /* l1_pae */
+ NULL, /* fl1_pae */
+#endif
+ NULL, /* l2_pae */
+ NULL, /* l2h_pae */
+ NULL, /* l3_pae */
+#if CONFIG_PAGING_LEVELS >= 4
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,4,4), /* l1_64 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_write_access,4,4), /* fl1_64 */
+#else
+ NULL, /* l1_64 */
+ NULL, /* fl1_64 */
+#endif
+ NULL, /* l2_64 */
+ NULL, /* l3_64 */
+ NULL, /* l4_64 */
+ NULL, /* p2m */
+ NULL /* unused */
+ };
+
+ static unsigned int callback_mask =
+ 1 << (PGC_SH2_l1_32_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_fl1_32_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_l1_pae_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_fl1_pae_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_l1_64_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_fl1_64_shadow >> PGC_SH2_type_shift)
+ ;
+ struct page_info *pg = mfn_to_page(gmfn);
+
+ ASSERT(shadow2_lock_is_acquired(v->domain));
+
+ /* Only remove writable mappings if we are doing shadow refcounts.
+ * In guest refcounting, we trust Xen to already be restricting
+ * all the writes to the guest page tables, so we do not need to
+ * do more. */
+ if ( !shadow2_mode_refcounts(v->domain) )
+ return 0;
+
+ /* Early exit if it's already a pagetable, or otherwise not writeable */
+ if ( sh2_mfn_is_a_page_table(gmfn)
+ || (pg->u.inuse.type_info & PGT_count_mask) == 0 )
+ return 0;
+
+ perfc_incrc(shadow2_writeable);
+
+ /* If this isn't a "normal" writeable page, the domain is trying to
+ * put pagetables in special memory of some kind. We can't allow that. */
+ if ( (pg->u.inuse.type_info & PGT_type_mask) != PGT_writable_page )
+ {
+ SHADOW2_ERROR("can't remove write access to mfn %lx, type_info is %"
+ PRtype_info "\n",
+ mfn_x(gmfn), mfn_to_page(gmfn)->u.inuse.type_info);
+ domain_crash(v->domain);
+ }
+
+#if SHADOW2_OPTIMIZATIONS & SH2OPT_WRITABLE_HEURISTIC
+ if ( v == current && level != 0 )
+ {
+ unsigned long gfn;
+ /* Heuristic: there is likely to be only one writeable mapping,
+ * and that mapping is likely to be in the current pagetable,
+ * either in the guest's linear map (linux, windows) or in a
+ * magic slot used to map high memory regions (linux HIGHTPTE) */
+
+#define GUESS(_a, _h) do { \
+ if ( v->arch.shadow2->guess_wrmap(v, (_a), gmfn) ) \
+ perfc_incrc(shadow2_writeable_h_ ## _h); \
+ if ( (pg->u.inuse.type_info & PGT_count_mask) == 0 ) \
+ return 1; \
+ } while (0)
+
+
+ /* Linux lowmem: first 1GB is mapped 1-to-1 above 0xC0000000 */
+ if ( v == current
+ && (gfn = sh2_mfn_to_gfn(v->domain, gmfn)) < 0x40000000 )
+ GUESS(0xC0000000 + (gfn << PAGE_SHIFT), 4);
+
+ if ( v->arch.shadow2->guest_levels == 2 )
+ {
+ if ( level == 1 )
+ /* 32bit non-PAE w2k3: linear map at 0xC0000000 */
+ GUESS(0xC0000000UL + (fault_addr >> 10), 1);
+ }
+#if CONFIG_PAGING_LEVELS >= 3
+ else if ( v->arch.shadow2->guest_levels == 3 )
+ {
+ /* 32bit PAE w2k3: linear map at 0xC0000000 */
+ switch ( level )
+ {
+ case 1: GUESS(0xC0000000UL + (fault_addr >> 9), 2); break;
+ case 2: GUESS(0xC0600000UL + (fault_addr >> 18), 2); break;
+ }
+ }
+#if CONFIG_PAGING_LEVELS >= 4
+ else if ( v->arch.shadow2->guest_levels == 4 )
+ {
+ /* 64bit w2k3: linear map at 0x0000070000000000 */
+ switch ( level )
+ {
+ case 1: GUESS(0x70000000000UL + (fault_addr >> 9), 3); break;
+ case 2: GUESS(0x70380000000UL + (fault_addr >> 18), 3); break;
+ case 3: GUESS(0x70381C00000UL + (fault_addr >> 27), 3); break;
+ }
+ }
+#endif /* CONFIG_PAGING_LEVELS >= 4 */
+#endif /* CONFIG_PAGING_LEVELS >= 3 */
+
+#undef GUESS
+
+ }
+#endif
+
+ /* Brute-force search of all the shadows, by walking the hash */
+ perfc_incrc(shadow2_writeable_bf);
+ hash_foreach(v, callback_mask, callbacks, gmfn);
+
+ /* If that didn't catch the mapping, something is very wrong */
+ if ( (mfn_to_page(gmfn)->u.inuse.type_info & PGT_count_mask) != 0 )
+ {
+ SHADOW2_ERROR("can't find all writeable mappings of mfn %lx: "
+ "%lu left\n", mfn_x(gmfn),
+ (mfn_to_page(gmfn)->u.inuse.type_info&PGT_count_mask));
+ domain_crash(v->domain);
+ }
+
+ /* We killed at least one writeable mapping, so must flush TLBs. */
+ return 1;
+}
+
+
+
+/**************************************************************************/
+/* Remove all mappings of a guest frame from the shadow tables.
+ * Returns non-zero if we need to flush TLBs. */
+
+int shadow2_remove_all_mappings(struct vcpu *v, mfn_t gmfn)
+{
+ struct page_info *page = mfn_to_page(gmfn);
+ int expected_count;
+
+ /* Dispatch table for getting per-type functions */
+ static hash_callback_t callbacks[16] = {
+ NULL, /* none */
+#if CONFIG_PAGING_LEVELS == 2
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,2,2), /* l1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,2,2), /* fl1_32 */
+#else
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,3,2), /* l1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,3,2), /* fl1_32 */
+#endif
+ NULL, /* l2_32 */
+#if CONFIG_PAGING_LEVELS >= 3
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,3,3), /* l1_pae */
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,3,3), /* fl1_pae */
+#else
+ NULL, /* l1_pae */
+ NULL, /* fl1_pae */
+#endif
+ NULL, /* l2_pae */
+ NULL, /* l2h_pae */
+ NULL, /* l3_pae */
+#if CONFIG_PAGING_LEVELS >= 4
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,4,4), /* l1_64 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_all_mappings,4,4), /* fl1_64 */
+#else
+ NULL, /* l1_64 */
+ NULL, /* fl1_64 */
+#endif
+ NULL, /* l2_64 */
+ NULL, /* l3_64 */
+ NULL, /* l4_64 */
+ NULL, /* p2m */
+ NULL /* unused */
+ };
+
+ static unsigned int callback_mask =
+ 1 << (PGC_SH2_l1_32_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_fl1_32_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_l1_pae_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_fl1_pae_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_l1_64_shadow >> PGC_SH2_type_shift)
+ | 1 << (PGC_SH2_fl1_64_shadow >> PGC_SH2_type_shift)
+ ;
+
+ perfc_incrc(shadow2_mappings);
+ if ( (page->count_info & PGC_count_mask) == 0 )
+ return 0;
+
+ ASSERT(shadow2_lock_is_acquired(v->domain));
+
+ /* XXX TODO:
+ * Heuristics for finding the (probably) single mapping of this gmfn */
+
+ /* Brute-force search of all the shadows, by walking the hash */
+ perfc_incrc(shadow2_mappings_bf);
+ hash_foreach(v, callback_mask, callbacks, gmfn);
+
+ /* If that didn't catch the mapping, something is very wrong */
+ expected_count = (page->count_info & PGC_allocated) ? 1 : 0;
+ if ( (page->count_info & PGC_count_mask) != expected_count )
+ {
+ /* Don't complain if we're in HVM and there's one extra mapping:
+ * The qemu helper process has an untyped mapping of this dom's RAM */
+ if ( !(shadow2_mode_external(v->domain)
+ && (page->count_info & PGC_count_mask) <= 2
+ && (page->u.inuse.type_info & PGT_count_mask) == 0) )
+ {
+ SHADOW2_ERROR("can't find all mappings of mfn %lx: "
+ "c=%08x t=%08lx\n", mfn_x(gmfn),
+ page->count_info, page->u.inuse.type_info);
+ }
+ }
+
+ /* We killed at least one mapping, so must flush TLBs. */
+ return 1;
+}
+
+
+/**************************************************************************/
+/* Remove all shadows of a guest frame from the shadow tables */
+
+static int sh2_remove_shadow_via_pointer(struct vcpu *v, mfn_t smfn)
+/* Follow this shadow's up-pointer, if it has one, and remove the reference
+ * found there. Returns 1 if that was the only reference to this shadow */
+{
+ struct page_info *pg = mfn_to_page(smfn);
+ mfn_t pmfn;
+ void *vaddr;
+ int rc;
+
+ ASSERT((pg->count_info & PGC_SH2_type_mask) > 0);
+ ASSERT((pg->count_info & PGC_SH2_type_mask) < PGC_SH2_max_shadow);
+ ASSERT((pg->count_info & PGC_SH2_type_mask) != PGC_SH2_l2_32_shadow);
+ ASSERT((pg->count_info & PGC_SH2_type_mask) != PGC_SH2_l3_pae_shadow);
+ ASSERT((pg->count_info & PGC_SH2_type_mask) != PGC_SH2_l4_64_shadow);
+
+ if (pg->up == 0) return 0;
+ pmfn = _mfn(pg->up >> PAGE_SHIFT);
+ ASSERT(valid_mfn(pmfn));
+ vaddr = sh2_map_domain_page(pmfn);
+ ASSERT(vaddr);
+ vaddr += pg->up & (PAGE_SIZE-1);
+ ASSERT(l1e_get_pfn(*(l1_pgentry_t *)vaddr) == mfn_x(smfn));
+
+ /* Is this the only reference to this shadow? */
+ rc = ((pg->count_info & PGC_SH2_count_mask) == 1) ? 1 : 0;
+
+ /* Blank the offending entry */
+ switch ((pg->count_info & PGC_SH2_type_mask))
+ {
+ case PGC_SH2_l1_32_shadow:
+ case PGC_SH2_l2_32_shadow:
+#if CONFIG_PAGING_LEVELS == 2
+ SHADOW2_INTERNAL_NAME(sh2_clear_shadow_entry,2,2)(v, vaddr, pmfn);
+#else
+ SHADOW2_INTERNAL_NAME(sh2_clear_shadow_entry,3,2)(v, vaddr, pmfn);
+#endif
+ break;
+#if CONFIG_PAGING_LEVELS >=3
+ case PGC_SH2_l1_pae_shadow:
+ case PGC_SH2_l2_pae_shadow:
+ case PGC_SH2_l2h_pae_shadow:
+ case PGC_SH2_l3_pae_shadow:
+ SHADOW2_INTERNAL_NAME(sh2_clear_shadow_entry,3,3)(v, vaddr, pmfn);
+ break;
+#if CONFIG_PAGING_LEVELS >= 4
+ case PGC_SH2_l1_64_shadow:
+ case PGC_SH2_l2_64_shadow:
+ case PGC_SH2_l3_64_shadow:
+ case PGC_SH2_l4_64_shadow:
+ SHADOW2_INTERNAL_NAME(sh2_clear_shadow_entry,4,4)(v, vaddr, pmfn);
+ break;
+#endif
+#endif
+ default: BUG(); /* Some wierd unknown shadow type */
+ }
+
+ sh2_unmap_domain_page(vaddr);
+ if ( rc )
+ perfc_incrc(shadow2_up_pointer);
+ else
+ perfc_incrc(shadow2_unshadow_bf);
+
+ return rc;
+}
+
+void sh2_remove_shadows(struct vcpu *v, mfn_t gmfn, int all)
+/* Remove the shadows of this guest page.
+ * If all != 0, find all shadows, if necessary by walking the tables.
+ * Otherwise, just try the (much faster) heuristics, which will remove
+ * at most one reference to each shadow of the page. */
+{
+ struct page_info *pg;
+ mfn_t smfn;
+ u32 sh_flags;
+ unsigned char t;
+
+ /* Dispatch table for getting per-type functions: each level must
+ * be called with the function to remove a lower-level shadow. */
+ static hash_callback_t callbacks[16] = {
+ NULL, /* none */
+ NULL, /* l1_32 */
+ NULL, /* fl1_32 */
+#if CONFIG_PAGING_LEVELS == 2
+ SHADOW2_INTERNAL_NAME(sh2_remove_l1_shadow,2,2), /* l2_32 */
+#else
+ SHADOW2_INTERNAL_NAME(sh2_remove_l1_shadow,3,2), /* l2_32 */
+#endif
+ NULL, /* l1_pae */
+ NULL, /* fl1_pae */
+#if CONFIG_PAGING_LEVELS >= 3
+ SHADOW2_INTERNAL_NAME(sh2_remove_l1_shadow,3,3), /* l2_pae */
+ SHADOW2_INTERNAL_NAME(sh2_remove_l1_shadow,3,3), /* l2h_pae */
+ SHADOW2_INTERNAL_NAME(sh2_remove_l2_shadow,3,3), /* l3_pae */
+#else
+ NULL, /* l2_pae */
+ NULL, /* l2h_pae */
+ NULL, /* l3_pae */
+#endif
+ NULL, /* l1_64 */
+ NULL, /* fl1_64 */
+#if CONFIG_PAGING_LEVELS >= 4
+ SHADOW2_INTERNAL_NAME(sh2_remove_l1_shadow,4,4), /* l2_64 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_l2_shadow,4,4), /* l3_64 */
+ SHADOW2_INTERNAL_NAME(sh2_remove_l3_shadow,4,4), /* l4_64 */
+#else
+ NULL, /* l2_64 */
+ NULL, /* l3_64 */
+ NULL, /* l4_64 */
+#endif
+ NULL, /* p2m */
+ NULL /* unused */
+ };
+
+ /* Another lookup table, for choosing which mask to use */
+ static unsigned int masks[16] = {
+ 0, /* none */
+ 1 << (PGC_SH2_l2_32_shadow >> PGC_SH2_type_shift), /* l1_32 */
+ 0, /* fl1_32 */
+ 0, /* l2_32 */
+ ((1 << (PGC_SH2_l2h_pae_shadow >> PGC_SH2_type_shift))
+ | (1 << (PGC_SH2_l2_pae_shadow >> PGC_SH2_type_shift))), /* l1_pae */
+ 0, /* fl1_pae */
+ 1 << (PGC_SH2_l3_pae_shadow >> PGC_SH2_type_shift), /* l2_pae */
+ 1 << (PGC_SH2_l3_pae_shadow >> PGC_SH2_type_shift), /* l2h_pae */
+ 0, /* l3_pae */
+ 1 << (PGC_SH2_l2_64_shadow >> PGC_SH2_type_shift), /* l1_64 */
+ 0, /* fl1_64 */
+ 1 << (PGC_SH2_l3_64_shadow >> PGC_SH2_type_shift), /* l2_64 */
+ 1 << (PGC_SH2_l4_64_shadow >> PGC_SH2_type_shift), /* l3_64 */
+ 0, /* l4_64 */
+ 0, /* p2m */
+ 0 /* unused */
+ };
+
+ SHADOW2_PRINTK("d=%d, v=%d, gmfn=%05lx\n",
+ v->domain->domain_id, v->vcpu_id, mfn_x(gmfn));
+
+ ASSERT(shadow2_lock_is_acquired(v->domain));
+
+ pg = mfn_to_page(gmfn);
+
+ /* Bale out now if the page is not shadowed */
+ if ( (pg->count_info & PGC_page_table) == 0 )
+ return;
+
+ /* Search for this shadow in all appropriate shadows */
+ perfc_incrc(shadow2_unshadow);
+ sh_flags = pg->shadow2_flags;
+
+ /* Lower-level shadows need to be excised from upper-level shadows.
+ * This call to hash_foreach() looks dangerous but is in fact OK: each
+ * call will remove at most one shadow, and terminate immediately when
+ * it does remove it, so we never walk the hash after doing a deletion. */
+#define DO_UNSHADOW(_type) do { \
+ t = (_type) >> PGC_SH2_type_shift; \
+ smfn = shadow2_hash_lookup(v, mfn_x(gmfn), t); \
+ if ( !sh2_remove_shadow_via_pointer(v, smfn) && all ) \
+ hash_foreach(v, masks[t], callbacks, smfn); \
+} while (0)
+
+ /* Top-level shadows need to be unpinned */
+#define DO_UNPIN(_type) do { \
+ t = (_type) >> PGC_SH2_type_shift; \
+ smfn = shadow2_hash_lookup(v, mfn_x(gmfn), t); \
+ if ( mfn_to_page(smfn)->count_info & PGC_SH2_pinned ) \
+ sh2_unpin(v, smfn); \
+ if ( (_type) == PGC_SH2_l3_pae_shadow ) \
+ SHADOW2_INTERNAL_NAME(sh2_unpin_all_l3_subshadows,3,3)(v, smfn); \
+} while (0)
+
+ if ( sh_flags & SH2F_L1_32 ) DO_UNSHADOW(PGC_SH2_l1_32_shadow);
+ if ( sh_flags & SH2F_L2_32 ) DO_UNPIN(PGC_SH2_l2_32_shadow);
+#if CONFIG_PAGING_LEVELS >= 3
+ if ( sh_flags & SH2F_L1_PAE ) DO_UNSHADOW(PGC_SH2_l1_pae_shadow);
+ if ( sh_flags & SH2F_L2_PAE ) DO_UNSHADOW(PGC_SH2_l2_pae_shadow);
+ if ( sh_flags & SH2F_L2H_PAE ) DO_UNSHADOW(PGC_SH2_l2h_pae_shadow);
+ if ( sh_flags & SH2F_L3_PAE ) DO_UNPIN(PGC_SH2_l3_pae_shadow);
+#if CONFIG_PAGING_LEVELS >= 4
+ if ( sh_flags & SH2F_L1_64 ) DO_UNSHADOW(PGC_SH2_l1_64_shadow);
+ if ( sh_flags & SH2F_L2_64 ) DO_UNSHADOW(PGC_SH2_l2_64_shadow);
+ if ( sh_flags & SH2F_L3_64 ) DO_UNSHADOW(PGC_SH2_l3_64_shadow);
+ if ( sh_flags & SH2F_L4_64 ) DO_UNPIN(PGC_SH2_l4_64_shadow);
+#endif
+#endif
+
+#undef DO_UNSHADOW
+#undef DO_UNPIN
+
+
+#if CONFIG_PAGING_LEVELS > 2
+ /* We may have caused some PAE l3 entries to change: need to
+ * fix up the copies of them in various places */
+ if ( sh_flags & (SH2F_L2_PAE|SH2F_L2H_PAE) )
+ sh2_pae_recopy(v->domain);
+#endif
+
+ /* If that didn't catch the shadows, something is wrong */
+ if ( all && (pg->count_info & PGC_page_table) )
+ {
+ SHADOW2_ERROR("can't find all shadows of mfn %05lx
(shadow2_flags=%08x)\n",
+ mfn_x(gmfn), pg->shadow2_flags);
+ domain_crash(v->domain);
+ }
+}
+
+void
+shadow2_remove_all_shadows_and_parents(struct vcpu *v, mfn_t gmfn)
+/* Even harsher: this is a HVM page that we thing is no longer a pagetable.
+ * Unshadow it, and recursively unshadow pages that reference it. */
+{
+ shadow2_remove_all_shadows(v, gmfn);
+ /* XXX TODO:
+ * Rework this hashtable walker to return a linked-list of all
+ * the shadows it modified, then do breadth-first recursion
+ * to find the way up to higher-level tables and unshadow them too.
+ *
+ * The current code (just tearing down each page's shadows as we
+ * detect that it is not a pagetable) is correct, but very slow.
+ * It means extra emulated writes and slows down removal of mappings. */
+}
+
+/**************************************************************************/
+
+void sh2_update_paging_modes(struct vcpu *v)
+{
+ struct domain *d = v->domain;
+ struct shadow2_entry_points *old_entries = v->arch.shadow2;
+ mfn_t old_guest_table;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+
+ // Valid transitions handled by this function:
+ // - For PV guests:
+ // - after a shadow mode has been changed
+ // - For HVM guests:
+ // - after a shadow mode has been changed
+ // - changes in CR0.PG, CR4.PAE, CR4.PSE, or CR4.PGE
+ //
+
+ // Avoid determining the current shadow2 mode for uninitialized CPUs, as
+ // we can not yet determine whether it is an HVM or PV domain.
+ //
+ if ( !test_bit(_VCPUF_initialised, &v->vcpu_flags) )
+ {
+ printk("%s: postponing determination of shadow2 mode\n", __func__);
+ return;
+ }
+
+ // First, tear down any old shadow tables held by this vcpu.
+ //
+ if ( v->arch.shadow2 )
+ shadow2_detach_old_tables(v);
+
+ if ( !hvm_guest(v) )
+ {
+ ///
+ /// PV guest
+ ///
+#if CONFIG_PAGING_LEVELS == 4
+ if ( pv_32bit_guest(v) )
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 4, 3);
+ else
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 4, 4);
+#elif CONFIG_PAGING_LEVELS == 3
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 3, 3);
+#elif CONFIG_PAGING_LEVELS == 2
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 2, 2);
+#else
+#error unexpected paging mode
+#endif
+ }
+ else
+ {
+ ///
+ /// HVM guest
+ ///
+ ASSERT(shadow2_mode_translate(d));
+ ASSERT(shadow2_mode_external(d));
+
+ if ( !hvm_paging_enabled(v) )
+ {
+ // paging disabled...
+ clear_bit(_VCPUF_shadow2_translate, &v->vcpu_flags);
+
+ /* Set v->arch.guest_table to use the p2m map, and choose
+ * the appropriate shadow mode */
+ old_guest_table = pagetable_get_mfn(v->arch.guest_table);
+#if CONFIG_PAGING_LEVELS == 2
+ v->arch.guest_table =
+ pagetable_from_pfn(pagetable_get_pfn(d->arch.phys_table));
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry,2,2);
+#elif CONFIG_PAGING_LEVELS == 3
+ v->arch.guest_table =
+ pagetable_from_pfn(pagetable_get_pfn(d->arch.phys_table));
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry,3,3);
+#else /* CONFIG_PAGING_LEVELS == 4 */
+ {
+ l4_pgentry_t *l4e;
+ /* Use the start of the first l3 table as a PAE l3 */
+ ASSERT(pagetable_get_pfn(d->arch.phys_table) != 0);
+ l4e =
sh2_map_domain_page(pagetable_get_mfn(d->arch.phys_table));
+ ASSERT(l4e_get_flags(l4e[0]) & _PAGE_PRESENT);
+ v->arch.guest_table =
+ pagetable_from_pfn(l4e_get_pfn(l4e[0]));
+ sh2_unmap_domain_page(l4e);
+ }
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry,3,3);
+#endif
+ /* Fix up refcounts on guest_table */
+ get_page(mfn_to_page(pagetable_get_mfn(v->arch.guest_table)), d);
+ if ( mfn_x(old_guest_table) != 0 )
+ put_page(mfn_to_page(old_guest_table));
+ }
+ else
+ {
+ set_bit(_VCPUF_shadow2_translate, &v->vcpu_flags);
+
+#ifdef __x86_64__
+ if ( hvm_long_mode_enabled(v) )
+ {
+ // long mode guest...
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 4, 4);
+ }
+ else
+#endif
+ if ( hvm_get_guest_ctrl_reg(v, 4) & X86_CR4_PAE )
+ {
+#if CONFIG_PAGING_LEVELS >= 3
+ // 32-bit PAE mode guest...
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 3,
3);
+#else
+ SHADOW2_ERROR("PAE not supported in 32-bit Xen\n");
+ domain_crash(d);
+ return;
+#endif
+ }
+ else
+ {
+ // 32-bit 2 level guest...
+#if CONFIG_PAGING_LEVELS >= 3
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 3,
2);
+#else
+ v->arch.shadow2 = &SHADOW2_INTERNAL_NAME(shadow2_entry, 2,
2);
+#endif
+ }
+ }
+
+ if ( pagetable_get_pfn(v->arch.monitor_table) == 0 )
+ {
+ mfn_t mmfn = shadow2_make_monitor_table(v);
+ v->arch.monitor_table = pagetable_from_mfn(mmfn);
+ v->arch.monitor_vtable = sh2_map_domain_page(mmfn);
+ }
+
+ if ( v->arch.shadow2 != old_entries )
+ {
+ SHADOW2_PRINTK("new paging mode: d=%u v=%u g=%u s=%u "
+ "(was g=%u s=%u)\n",
+ d->domain_id, v->vcpu_id,
+ v->arch.shadow2->guest_levels,
+ v->arch.shadow2->shadow_levels,
+ old_entries ? old_entries->guest_levels : 0,
+ old_entries ? old_entries->shadow_levels : 0);
+ if ( old_entries &&
+ (v->arch.shadow2->shadow_levels !=
+ old_entries->shadow_levels) )
+ {
+ /* Need to make a new monitor table for the new mode */
+ mfn_t new_mfn, old_mfn;
+
+ if ( v != current )
+ {
+ SHADOW2_ERROR("Some third party (d=%u v=%u) is changing "
+ "this HVM vcpu's (d=%u v=%u) paging mode!\n",
+ current->domain->domain_id, current->vcpu_id,
+ v->domain->domain_id, v->vcpu_id);
+ domain_crash(v->domain);
+ return;
+ }
+
+ sh2_unmap_domain_page(v->arch.monitor_vtable);
+ old_mfn = pagetable_get_mfn(v->arch.monitor_table);
+ v->arch.monitor_table = pagetable_null();
+ new_mfn = v->arch.shadow2->make_monitor_table(v);
+ v->arch.monitor_table = pagetable_from_mfn(new_mfn);
+ v->arch.monitor_vtable = sh2_map_domain_page(new_mfn);
+ SHADOW2_PRINTK("new monitor table %"SH2_PRI_mfn "\n",
+ mfn_x(new_mfn));
+
+ /* Don't be running on the old monitor table when we
+ * pull it down! Switch CR3, and warn the HVM code that
+ * its host cr3 has changed. */
+ make_cr3(v, mfn_x(new_mfn));
+ write_ptbase(v);
+ hvm_update_host_cr3(v);
+ old_entries->destroy_monitor_table(v, old_mfn);
+ }
+ }
+
+ // XXX -- Need to deal with changes in CR4.PSE and CR4.PGE.
+ // These are HARD: think about the case where two CPU's have
+ // different values for CR4.PSE and CR4.PGE at the same time.
+ // This *does* happen, at least for CR4.PGE...
+ }
+
+ v->arch.shadow2->update_cr3(v);
+}
+
+/**************************************************************************/
+/* Turning on and off shadow2 features */
+
+static void sh2_new_mode(struct domain *d, u32 new_mode)
+/* Inform all the vcpus that the shadow mode has been changed */
+{
+ struct vcpu *v;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(d != current->domain);
+ d->arch.shadow2_mode = new_mode;
+ if ( new_mode & SHM2_translate )
+ shadow2_audit_p2m(d);
+ for_each_vcpu(d, v)
+ sh2_update_paging_modes(v);
+}
+
+static int shadow2_enable(struct domain *d, u32 mode)
+/* Turn on "permanent" shadow features: external, translate, refcount.
+ * Can only be called once on a domain, and these features cannot be
+ * disabled.
+ * Returns 0 for success, -errno for failure. */
+{
+ unsigned int old_pages;
+ int rv = 0;
+
+ domain_pause(d);
+ shadow2_lock(d);
+
+ /* Sanity check the arguments */
+ if ( d == current->domain
+ || shadow2_mode_enabled(d)
+ || !(mode & SHM2_enable)
+ || ((mode & SHM2_external) && !(mode & SHM2_translate)) )
+ {
+ rv = -EINVAL;
+ goto out;
+ }
+
+ // XXX -- eventually would like to require that all memory be allocated
+ // *after* shadow2_enabled() is called... So here, we would test to make
+ // sure that d->page_list is empty.
+#if 0
+ spin_lock(&d->page_alloc_lock);
+ if ( !list_empty(&d->page_list) )
+ {
+ spin_unlock(&d->page_alloc_lock);
+ rv = -EINVAL;
+ goto out;
+ }
+ spin_unlock(&d->page_alloc_lock);
+#endif
+
+ /* Init the shadow memory allocation if the user hasn't done so */
+ old_pages = d->arch.shadow2_total_pages;
+ if ( old_pages == 0 )
+ if ( set_sh2_allocation(d, 256, NULL) != 0 ) /* Use at least 1MB */
+ {
+ set_sh2_allocation(d, 0, NULL);
+ rv = -ENOMEM;
+ goto out;
+ }
+
+ /* Init the hash table */
+ if ( shadow2_hash_alloc(d) != 0 )
+ {
+ set_sh2_allocation(d, old_pages, NULL);
+ rv = -ENOMEM;
+ goto out;
+ }
+
+ /* Init the P2M table */
+ if ( mode & SHM2_translate )
+ if ( !shadow2_alloc_p2m_table(d) )
+ {
+ shadow2_hash_teardown(d);
+ set_sh2_allocation(d, old_pages, NULL);
+ shadow2_p2m_teardown(d);
+ rv = -ENOMEM;
+ goto out;
+ }
+
+ /* Update the bits */
+ sh2_new_mode(d, mode);
+ shadow2_audit_p2m(d);
+ out:
+ shadow2_unlock(d);
+ domain_unpause(d);
+ return 0;
+}
+
+void shadow2_teardown(struct domain *d)
+/* Destroy the shadow pagetables of this domain and free its shadow memory.
+ * Should only be called for dying domains. */
+{
+ struct vcpu *v;
+ mfn_t mfn;
+
+ ASSERT(test_bit(_DOMF_dying, &d->domain_flags));
+ ASSERT(d != current->domain);
+
+ if ( !shadow2_lock_is_acquired(d) )
+ shadow2_lock(d); /* Keep various asserts happy */
+
+ if ( shadow2_mode_enabled(d) )
+ {
+ /* Release the shadow and monitor tables held by each vcpu */
+ for_each_vcpu(d, v)
+ {
+ if ( v->arch.shadow2 )
+ shadow2_detach_old_tables(v);
+ if ( shadow2_mode_external(d) )
+ {
+ mfn = pagetable_get_mfn(v->arch.monitor_table);
+ if ( valid_mfn(mfn) && (mfn_x(mfn) != 0) )
+ shadow2_destroy_monitor_table(v, mfn);
+ v->arch.monitor_table = pagetable_null();
+ }
+ }
+ }
+
+ if ( d->arch.shadow2_total_pages != 0 )
+ {
+ SHADOW2_PRINTK("teardown of domain %u starts."
+ " Shadow pages total = %u, free = %u, p2m=%u\n",
+ d->domain_id,
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+ /* Destroy all the shadows and release memory to domheap */
+ set_sh2_allocation(d, 0, NULL);
+ /* Release the hash table back to xenheap */
+ if (d->arch.shadow2_hash_table)
+ shadow2_hash_teardown(d);
+ /* Release the log-dirty bitmap of dirtied pages */
+ sh2_free_log_dirty_bitmap(d);
+ /* Should not have any more memory held */
+ SHADOW2_PRINTK("teardown done."
+ " Shadow pages total = %u, free = %u, p2m=%u\n",
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+ ASSERT(d->arch.shadow2_total_pages == 0);
+ }
+
+ /* We leave the "permanent" shadow modes enabled, but clear the
+ * log-dirty mode bit. We don't want any more mark_dirty()
+ * calls now that we've torn down the bitmap */
+ d->arch.shadow2_mode &= ~SHM2_log_dirty;
+
+ shadow2_unlock(d);
+}
+
+void shadow2_final_teardown(struct domain *d)
+/* Called by arch_domain_destroy(), when it's safe to pull down the p2m map. */
+{
+
+ SHADOW2_PRINTK("dom %u final teardown starts."
+ " Shadow pages total = %u, free = %u, p2m=%u\n",
+ d->domain_id,
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+
+ /* Double-check that the domain didn't have any shadow memory.
+ * It is possible for a domain that never got domain_kill()ed
+ * to get here with its shadow allocation intact. */
+ if ( d->arch.shadow2_total_pages != 0 )
+ shadow2_teardown(d);
+
+ /* It is now safe to pull down the p2m map. */
+ if ( d->arch.shadow2_p2m_pages != 0 )
+ shadow2_p2m_teardown(d);
+
+ SHADOW2_PRINTK("dom %u final teardown done."
+ " Shadow pages total = %u, free = %u, p2m=%u\n",
+ d->domain_id,
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+}
+
+static int shadow2_one_bit_enable(struct domain *d, u32 mode)
+/* Turn on a single shadow mode feature */
+{
+ ASSERT(shadow2_lock_is_acquired(d));
+
+ /* Sanity check the call */
+ if ( d == current->domain || (d->arch.shadow2_mode & mode) )
+ {
+ return -EINVAL;
+ }
+
+ if ( d->arch.shadow2_mode == 0 )
+ {
+ /* Init the shadow memory allocation and the hash table */
+ if ( set_sh2_allocation(d, 1, NULL) != 0
+ || shadow2_hash_alloc(d) != 0 )
+ {
+ set_sh2_allocation(d, 0, NULL);
+ return -ENOMEM;
+ }
+ }
+
+ /* Update the bits */
+ sh2_new_mode(d, d->arch.shadow2_mode | mode);
+
+ return 0;
+}
+
+static int shadow2_one_bit_disable(struct domain *d, u32 mode)
+/* Turn off a single shadow mode feature */
+{
+ struct vcpu *v;
+ ASSERT(shadow2_lock_is_acquired(d));
+
+ /* Sanity check the call */
+ if ( d == current->domain || !(d->arch.shadow2_mode & mode) )
+ {
+ return -EINVAL;
+ }
+
+ /* Update the bits */
+ sh2_new_mode(d, d->arch.shadow2_mode & ~mode);
+ if ( d->arch.shadow2_mode == 0 )
+ {
+ /* Get this domain off shadows */
+ SHADOW2_PRINTK("un-shadowing of domain %u starts."
+ " Shadow pages total = %u, free = %u, p2m=%u\n",
+ d->domain_id,
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+ for_each_vcpu(d, v)
+ {
+ if ( v->arch.shadow2 )
+ shadow2_detach_old_tables(v);
+#if CONFIG_PAGING_LEVELS == 4
+ if ( !(v->arch.flags & TF_kernel_mode) )
+ make_cr3(v, pagetable_get_pfn(v->arch.guest_table_user));
+ else
+#endif
+ make_cr3(v, pagetable_get_pfn(v->arch.guest_table));
+
+ }
+
+ /* Pull down the memory allocation */
+ if ( set_sh2_allocation(d, 0, NULL) != 0 )
+ {
+ // XXX - How can this occur?
+ // Seems like a bug to return an error now that we've
+ // disabled the relevant shadow mode.
+ //
+ return -ENOMEM;
+ }
+ shadow2_hash_teardown(d);
+ SHADOW2_PRINTK("un-shadowing of domain %u done."
+ " Shadow pages total = %u, free = %u, p2m=%u\n",
+ d->domain_id,
+ d->arch.shadow2_total_pages,
+ d->arch.shadow2_free_pages,
+ d->arch.shadow2_p2m_pages);
+ }
+
+ return 0;
+}
+
+/* Enable/disable ops for the "test" and "log-dirty" modes */
+int shadow2_test_enable(struct domain *d)
+{
+ int ret;
+
+ domain_pause(d);
+ shadow2_lock(d);
+
+ if ( shadow2_mode_enabled(d) )
+ {
+ SHADOW2_ERROR("Don't support enabling test mode"
+ "on already shadowed doms\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = shadow2_one_bit_enable(d, SHM2_enable);
+ out:
+ shadow2_unlock(d);
+ domain_unpause(d);
+
+ return ret;
+}
+
+int shadow2_test_disable(struct domain *d)
+{
+ int ret;
+
+ domain_pause(d);
+ shadow2_lock(d);
+ ret = shadow2_one_bit_disable(d, SHM2_enable);
+ shadow2_unlock(d);
+ domain_unpause(d);
+
+ return ret;
+}
+
+static int
+sh2_alloc_log_dirty_bitmap(struct domain *d)
+{
+ ASSERT(d->arch.shadow_dirty_bitmap == NULL);
+ d->arch.shadow_dirty_bitmap_size =
+ (d->shared_info->arch.max_pfn + (BITS_PER_LONG - 1)) &
+ ~(BITS_PER_LONG - 1);
+ d->arch.shadow_dirty_bitmap =
+ xmalloc_array(unsigned long,
+ d->arch.shadow_dirty_bitmap_size / BITS_PER_LONG);
+ if ( d->arch.shadow_dirty_bitmap == NULL )
+ {
+ d->arch.shadow_dirty_bitmap_size = 0;
+ return -ENOMEM;
+ }
+ memset(d->arch.shadow_dirty_bitmap, 0, d->arch.shadow_dirty_bitmap_size/8);
+
+ return 0;
+}
+
+static void
+sh2_free_log_dirty_bitmap(struct domain *d)
+{
+ d->arch.shadow_dirty_bitmap_size = 0;
+ if ( d->arch.shadow_dirty_bitmap )
+ {
+ xfree(d->arch.shadow_dirty_bitmap);
+ d->arch.shadow_dirty_bitmap = NULL;
+ }
+}
+
+static int shadow2_log_dirty_enable(struct domain *d)
+{
+ int ret;
+
+ domain_pause(d);
+ shadow2_lock(d);
+
+ if ( shadow2_mode_log_dirty(d) )
+ {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if ( shadow2_mode_enabled(d) )
+ {
+ SHADOW2_ERROR("Don't (yet) support enabling log-dirty"
+ "on already shadowed doms\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = sh2_alloc_log_dirty_bitmap(d);
+ if ( ret != 0 )
+ {
+ sh2_free_log_dirty_bitmap(d);
+ goto out;
+ }
+
+ ret = shadow2_one_bit_enable(d, SHM2_log_dirty);
+ if ( ret != 0 )
+ sh2_free_log_dirty_bitmap(d);
+
+ out:
+ shadow2_unlock(d);
+ domain_unpause(d);
+ return ret;
+}
+
+static int shadow2_log_dirty_disable(struct domain *d)
+{
+ int ret;
+
+ domain_pause(d);
+ shadow2_lock(d);
+ ret = shadow2_one_bit_disable(d, SHM2_log_dirty);
+ if ( !shadow2_mode_log_dirty(d) )
+ sh2_free_log_dirty_bitmap(d);
+ shadow2_unlock(d);
+ domain_unpause(d);
+
+ return ret;
+}
+
+/**************************************************************************/
+/* P2M map manipulations */
+
+static void
+sh2_p2m_remove_page(struct domain *d, unsigned long gfn, unsigned long mfn)
+{
+ struct vcpu *v;
+
+ if ( !shadow2_mode_translate(d) )
+ return;
+
+ v = current;
+ if ( v->domain != d )
+ v = d->vcpu[0];
+
+
+ SHADOW2_PRINTK("removing gfn=%#lx mfn=%#lx\n", gfn, mfn);
+
+ ASSERT(mfn_x(sh2_gfn_to_mfn(d, gfn)) == mfn);
+ //ASSERT(sh2_mfn_to_gfn(d, mfn) == gfn);
+
+ shadow2_remove_all_shadows_and_parents(v, _mfn(mfn));
+ if ( shadow2_remove_all_mappings(v, _mfn(mfn)) )
+ flush_tlb_mask(d->domain_dirty_cpumask);
+ shadow2_set_p2m_entry(d, gfn, _mfn(INVALID_MFN));
+ set_gpfn_from_mfn(mfn, INVALID_M2P_ENTRY);
+}
+
+void
+shadow2_guest_physmap_remove_page(struct domain *d, unsigned long gfn,
+ unsigned long mfn)
+{
+ shadow2_lock(d);
+ shadow2_audit_p2m(d);
+ sh2_p2m_remove_page(d, gfn, mfn);
+ shadow2_audit_p2m(d);
+ shadow2_unlock(d);
+}
+
+void
+shadow2_guest_physmap_add_page(struct domain *d, unsigned long gfn,
+ unsigned long mfn)
+{
+ struct vcpu *v;
+ unsigned long ogfn;
+ mfn_t omfn;
+
+ if ( !shadow2_mode_translate(d) )
+ return;
+
+ v = current;
+ if ( v->domain != d )
+ v = d->vcpu[0];
+
+ shadow2_lock(d);
+ shadow2_audit_p2m(d);
+
+ SHADOW2_DEBUG(P2M, "adding gfn=%#lx mfn=%#lx\n", gfn, mfn);
+
+ omfn = sh2_gfn_to_mfn(d, gfn);
+ if ( valid_mfn(omfn) )
+ {
+ /* Get rid of the old mapping, especially any shadows */
+ shadow2_remove_all_shadows_and_parents(v, omfn);
+ if ( shadow2_remove_all_mappings(v, omfn) )
+ flush_tlb_mask(d->domain_dirty_cpumask);
+ set_gpfn_from_mfn(mfn_x(omfn), INVALID_M2P_ENTRY);
+ }
+
+ ogfn = sh2_mfn_to_gfn(d, _mfn(mfn));
+ if (
+#ifdef __x86_64__
+ (ogfn != 0x5555555555555555L)
+#else
+ (ogfn != 0x55555555L)
+#endif
+ && (ogfn != INVALID_M2P_ENTRY)
+ && (ogfn != gfn) )
+ {
+ /* This machine frame is already mapped at another physical address */
+ SHADOW2_DEBUG(P2M, "aliased! mfn=%#lx, old gfn=%#lx, new gfn=%#lx\n",
+ mfn, ogfn, gfn);
+ if ( valid_mfn(omfn = sh2_gfn_to_mfn(d, ogfn)) )
+ {
+ SHADOW2_DEBUG(P2M, "old gfn=%#lx -> mfn %#lx\n",
+ ogfn , mfn_x(omfn));
+ if ( mfn_x(omfn) == mfn )
+ sh2_p2m_remove_page(d, ogfn, mfn);
+ }
+ }
+
+ shadow2_set_p2m_entry(d, gfn, _mfn(mfn));
+ set_gpfn_from_mfn(mfn, gfn);
+ shadow2_audit_p2m(d);
+ shadow2_unlock(d);
+}
+
+/**************************************************************************/
+/* Log-dirty mode support */
+
+/* Convert a shadow to log-dirty mode. */
+void shadow2_convert_to_log_dirty(struct vcpu *v, mfn_t smfn)
+{
+ BUG();
+}
+
+
+/* Read a domain's log-dirty bitmap and stats.
+ * If the operation is a CLEAN, clear the bitmap and stats as well. */
+static int shadow2_log_dirty_op(struct domain *d, dom0_shadow_control_t *sc)
+{
+ int i, rv = 0, clean = 0;
+
+ domain_pause(d);
+ shadow2_lock(d);
+
+ if ( sc->op == DOM0_SHADOW_CONTROL_OP_CLEAN
+ || sc->op == DOM0_SHADOW_CONTROL_OP_FLUSH )
+ clean = 1;
+ else
+ ASSERT(sc->op == DOM0_SHADOW_CONTROL_OP_PEEK);
+
+ SHADOW2_DEBUG(LOGDIRTY, "log-dirty %s: dom %u faults=%u dirty=%u\n",
+ (clean) ? "clean" : "peek",
+ d->domain_id,
+ d->arch.shadow_fault_count,
+ d->arch.shadow_dirty_count);
+
+ sc->stats.fault_count = d->arch.shadow_fault_count;
+ sc->stats.dirty_count = d->arch.shadow_dirty_count;
+
+ if ( clean )
+ {
+ struct list_head *l, *t;
+ struct page_info *pg;
+
+ /* Need to revoke write access to the domain's pages again.
+ * In future, we'll have a less heavy-handed approach to this,
+ * but for now, we just unshadow everything except Xen. */
+ list_for_each_safe(l, t, &d->arch.shadow2_toplevel_shadows)
+ {
+ pg = list_entry(l, struct page_info, list);
+ shadow2_unhook_mappings(d->vcpu[0], page_to_mfn(pg));
+ }
+
+ d->arch.shadow_fault_count = 0;
+ d->arch.shadow_dirty_count = 0;
+ }
+
+ if ( guest_handle_is_null(sc->dirty_bitmap) ||
+ (d->arch.shadow_dirty_bitmap == NULL) )
+ {
+ rv = -EINVAL;
+ goto out;
+ }
+
+ if ( sc->pages > d->arch.shadow_dirty_bitmap_size )
+ sc->pages = d->arch.shadow_dirty_bitmap_size;
+
+#define CHUNK (8*1024) /* Transfer and clear in 1kB chunks for L1 cache. */
+ for ( i = 0; i < sc->pages; i += CHUNK )
+ {
+ int bytes = ((((sc->pages - i) > CHUNK)
+ ? CHUNK
+ : (sc->pages - i)) + 7) / 8;
+
+ if ( copy_to_guest_offset(
+ sc->dirty_bitmap,
+ i/(8*sizeof(unsigned long)),
+ d->arch.shadow_dirty_bitmap + (i/(8*sizeof(unsigned long))),
+ (bytes + sizeof(unsigned long) - 1) / sizeof(unsigned long)) )
+ {
+ rv = -EINVAL;
+ goto out;
+ }
+
+ if ( clean )
+ memset(d->arch.shadow_dirty_bitmap + (i/(8*sizeof(unsigned long))),
+ 0, bytes);
+ }
+#undef CHUNK
+
+ out:
+ shadow2_unlock(d);
+ domain_unpause(d);
+ return 0;
+}
+
+
+/* Mark a page as dirty */
+void sh2_do_mark_dirty(struct domain *d, mfn_t gmfn)
+{
+ unsigned long pfn;
+
+ ASSERT(shadow2_lock_is_acquired(d));
+ ASSERT(shadow2_mode_log_dirty(d));
+
+ if ( !valid_mfn(gmfn) )
+ return;
+
+ ASSERT(d->arch.shadow_dirty_bitmap != NULL);
+
+ /* We /really/ mean PFN here, even for non-translated guests. */
+ pfn = get_gpfn_from_mfn(mfn_x(gmfn));
+
+ /*
+ * Values with the MSB set denote MFNs that aren't really part of the
+ * domain's pseudo-physical memory map (e.g., the shared info frame).
+ * Nothing to do here...
+ */
+ if ( unlikely(!VALID_M2P(pfn)) )
+ return;
+
+ /* N.B. Can use non-atomic TAS because protected by shadow2_lock. */
+ if ( likely(pfn < d->arch.shadow_dirty_bitmap_size) )
+ {
+ if ( !__test_and_set_bit(pfn, d->arch.shadow_dirty_bitmap) )
+ {
+ SHADOW2_DEBUG(LOGDIRTY,
+ "marked mfn %" SH2_PRI_mfn " (pfn=%lx), dom %d\n",
+ mfn_x(gmfn), pfn, d->domain_id);
+ d->arch.shadow_dirty_count++;
+ }
+ }
+ else
+ {
+ SHADOW2_PRINTK("mark_dirty OOR! "
+ "mfn=%" SH2_PRI_mfn " pfn=%lx max=%x (dom %d)\n"
+ "owner=%d c=%08x t=%" PRtype_info "\n",
+ mfn_x(gmfn),
+ pfn,
+ d->arch.shadow_dirty_bitmap_size,
+ d->domain_id,
+ (page_get_owner(mfn_to_page(gmfn))
+ ? page_get_owner(mfn_to_page(gmfn))->domain_id
+ : -1),
+ mfn_to_page(gmfn)->count_info,
+ mfn_to_page(gmfn)->u.inuse.type_info);
+ }
+}
+
+
+/**************************************************************************/
+/* Shadow-control DOM0_OP dispatcher */
+
+int shadow2_control_op(struct domain *d,
+ dom0_shadow_control_t *sc,
+ XEN_GUEST_HANDLE(dom0_op_t) u_dom0_op)
+{
+ int rc, preempted = 0;
+
+ if ( unlikely(d == current->domain) )
+ {
+ DPRINTK("Don't try to do a shadow op on yourself!\n");
+ return -EINVAL;
+ }
+
+ switch ( sc->op )
+ {
+ case DOM0_SHADOW_CONTROL_OP_OFF:
+ if ( shadow2_mode_log_dirty(d) )
+ if ( (rc = shadow2_log_dirty_disable(d)) != 0 )
+ return rc;
+ if ( d->arch.shadow2_mode & SHM2_enable )
+ if ( (rc = shadow2_test_disable(d)) != 0 )
+ return rc;
+ return 0;
+
+ case DOM0_SHADOW_CONTROL_OP_ENABLE_TEST:
+ return shadow2_test_enable(d);
+
+ case DOM0_SHADOW_CONTROL_OP_ENABLE_LOGDIRTY:
+ return shadow2_log_dirty_enable(d);
+
+ case DOM0_SHADOW_CONTROL_OP_FLUSH:
+ case DOM0_SHADOW_CONTROL_OP_CLEAN:
+ case DOM0_SHADOW_CONTROL_OP_PEEK:
+ return shadow2_log_dirty_op(d, sc);
+
+
+
+ case DOM0_SHADOW2_CONTROL_OP_ENABLE:
+ return shadow2_enable(d, sc->mode << SHM2_shift);
+
+ case DOM0_SHADOW2_CONTROL_OP_GET_ALLOCATION:
+ sc->mb = shadow2_get_allocation(d);
+ return 0;
+
+ case DOM0_SHADOW2_CONTROL_OP_SET_ALLOCATION:
+ rc = shadow2_set_allocation(d, sc->mb, &preempted);
+ if ( preempted )
+ /* Not finished. Set up to re-run the call. */
+ rc = hypercall_create_continuation(
+ __HYPERVISOR_dom0_op, "h", u_dom0_op);
+ else
+ /* Finished. Return the new allocation */
+ sc->mb = shadow2_get_allocation(d);
+ return rc;
+
+
+ default:
+ SHADOW2_ERROR("Bad shadow op %u\n", sc->op);
+ return -EINVAL;
+ }
+}
+
+
+/**************************************************************************/
+/* Auditing shadow tables */
+
+#if SHADOW2_AUDIT & SHADOW2_AUDIT_ENTRIES_FULL
+
+void shadow2_audit_tables(struct vcpu *v)
+{
+ /* Dispatch table for getting per-type functions */
+ static hash_callback_t callbacks[16] = {
+ NULL, /* none */
+#if CONFIG_PAGING_LEVELS == 2
+ SHADOW2_INTERNAL_NAME(sh2_audit_l1_table,2,2), /* l1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_fl1_table,2,2), /* fl1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l2_table,2,2), /* l2_32 */
+#else
+ SHADOW2_INTERNAL_NAME(sh2_audit_l1_table,3,2), /* l1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_fl1_table,3,2), /* fl1_32 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l2_table,3,2), /* l2_32 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l1_table,3,3), /* l1_pae */
+ SHADOW2_INTERNAL_NAME(sh2_audit_fl1_table,3,3), /* fl1_pae */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l2_table,3,3), /* l2_pae */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l2_table,3,3), /* l2h_pae */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l3_table,3,3), /* l3_pae */
+#if CONFIG_PAGING_LEVELS >= 4
+ SHADOW2_INTERNAL_NAME(sh2_audit_l1_table,4,4), /* l1_64 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_fl1_table,4,4), /* fl1_64 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l2_table,4,4), /* l2_64 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l3_table,4,4), /* l3_64 */
+ SHADOW2_INTERNAL_NAME(sh2_audit_l4_table,4,4), /* l4_64 */
+#endif /* CONFIG_PAGING_LEVELS >= 4 */
+#endif /* CONFIG_PAGING_LEVELS > 2 */
+ NULL /* All the rest */
+ };
+ unsigned int mask;
+
+ if ( !(SHADOW2_AUDIT_ENABLE) )
+ return;
+
+ if ( SHADOW2_AUDIT & SHADOW2_AUDIT_ENTRIES_FULL )
+ mask = ~1; /* Audit every table in the system */
+ else
+ {
+ /* Audit only the current mode's tables */
+ switch (v->arch.shadow2->guest_levels)
+ {
+ case 2: mask = (SH2F_L1_32|SH2F_FL1_32|SH2F_L2_32); break;
+ case 3: mask = (SH2F_L1_PAE|SH2F_FL1_PAE|SH2F_L2_PAE
+ |SH2F_L2H_PAE|SH2F_L3_PAE); break;
+ case 4: mask = (SH2F_L1_64|SH2F_FL1_64|SH2F_L2_64
+ |SH2F_L3_64|SH2F_L4_64); break;
+ default: BUG();
+ }
+ }
+
+ hash_foreach(v, ~1, callbacks, _mfn(INVALID_MFN));
+}
+
+#endif /* Shadow audit */
+
+
+/**************************************************************************/
+/* Auditing p2m tables */
+
+#if SHADOW2_AUDIT & SHADOW2_AUDIT_P2M
+
+void shadow2_audit_p2m(struct domain *d)
+{
+ struct list_head *entry;
+ struct page_info *page;
+ struct domain *od;
+ unsigned long mfn, gfn, m2pfn, lp2mfn = 0;
+ mfn_t p2mfn;
+ unsigned long orphans_d = 0, orphans_i = 0, mpbad = 0, pmbad = 0;
+ int test_linear;
+
+ if ( !(SHADOW2_AUDIT_ENABLE) || !shadow2_mode_translate(d) )
+ return;
+
+ //SHADOW2_PRINTK("p2m audit starts\n");
+
+ test_linear = ( (d == current->domain) && current->arch.monitor_vtable );
+ if ( test_linear )
+ local_flush_tlb();
+
+ /* Audit part one: walk the domain's page allocation list, checking
+ * the m2p entries. */
+ for ( entry = d->page_list.next;
+ entry != &d->page_list;
+ entry = entry->next )
+ {
+ page = list_entry(entry, struct page_info, list);
+ mfn = mfn_x(page_to_mfn(page));
+
+ // SHADOW2_PRINTK("auditing guest page, mfn=%#lx\n", mfn);
+
+ od = page_get_owner(page);
+
+ if ( od != d )
+ {
+ SHADOW2_PRINTK("wrong owner %#lx -> %p(%u) != %p(%u)\n",
+ mfn, od, (od?od->domain_id:-1), d, d->domain_id);
+ continue;
+ }
+
+ gfn = get_gpfn_from_mfn(mfn);
+ if ( gfn == INVALID_M2P_ENTRY )
+ {
+ orphans_i++;
+ //SHADOW2_PRINTK("orphaned guest page: mfn=%#lx has invalid gfn\n",
+ // mfn);
+ continue;
+ }
+
+ if ( gfn == 0x55555555 )
+ {
+ orphans_d++;
+ //SHADOW2_PRINTK("orphaned guest page: mfn=%#lx has debug gfn\n",
+ // mfn);
+ continue;
+ }
+
+ p2mfn = sh2_gfn_to_mfn_foreign(d, gfn);
+ if ( mfn_x(p2mfn) != mfn )
+ {
+ mpbad++;
+ SHADOW2_PRINTK("map mismatch mfn %#lx -> gfn %#lx -> mfn %#lx"
+ " (-> gfn %#lx)\n",
+ mfn, gfn, mfn_x(p2mfn),
+ (mfn_valid(p2mfn)
+ ? get_gpfn_from_mfn(mfn_x(p2mfn))
+ : -1u));
+ /* This m2p entry is stale: the domain has another frame in
+ * this physical slot. No great disaster, but for neatness,
+ * blow away the m2p entry. */
+ set_gpfn_from_mfn(mfn, INVALID_M2P_ENTRY);
+ }
+
+ if ( test_linear )
+ {
+ lp2mfn = get_mfn_from_gpfn(gfn);
+ if ( lp2mfn != mfn_x(p2mfn) )
+ {
+ SHADOW2_PRINTK("linear mismatch gfn %#lx -> mfn %#lx "
+ "(!= mfn %#lx)\n", gfn, lp2mfn, p2mfn);
+ }
+ }
+
+ // SHADOW2_PRINTK("OK: mfn=%#lx, gfn=%#lx, p2mfn=%#lx, lp2mfn=%#lx\n",
+ // mfn, gfn, p2mfn, lp2mfn);
+ }
+
+ /* Audit part two: walk the domain's p2m table, checking the entries. */
+ if ( pagetable_get_pfn(d->arch.phys_table) != 0 )
+ {
+ l2_pgentry_t *l2e;
+ l1_pgentry_t *l1e;
+ int i1, i2;
+
+#if CONFIG_PAGING_LEVELS == 4
+ l4_pgentry_t *l4e;
+ l3_pgentry_t *l3e;
+ int i3, i4;
+ l4e = sh2_map_domain_page(pagetable_get_mfn(d->arch.phys_table));
+#elif CONFIG_PAGING_LEVELS == 3
+ l3_pgentry_t *l3e;
+ int i3;
+ l3e = sh2_map_domain_page(pagetable_get_mfn(d->arch.phys_table));
+#else /* CONFIG_PAGING_LEVELS == 2 */
+ l2e = sh2_map_domain_page(pagetable_get_mfn(d->arch.phys_table));
+#endif
+
+ gfn = 0;
+#if CONFIG_PAGING_LEVELS >= 3
+#if CONFIG_PAGING_LEVELS >= 4
+ for ( i4 = 0; i4 < L4_PAGETABLE_ENTRIES; i4++ )
+ {
+ if ( !(l4e_get_flags(l4e[i4]) & _PAGE_PRESENT) )
+ {
+ gfn += 1 << (L4_PAGETABLE_SHIFT - PAGE_SHIFT);
+ continue;
+ }
+ l3e = sh2_map_domain_page(_mfn(l4e_get_pfn(l4e[i4])));
+#endif /* now at levels 3 or 4... */
+ for ( i3 = 0;
+ i3 < ((CONFIG_PAGING_LEVELS==4) ? L3_PAGETABLE_ENTRIES : 8);
+ i3++ )
+ {
+ if ( !(l3e_get_flags(l3e[i3]) & _PAGE_PRESENT) )
+ {
+ gfn += 1 << (L3_PAGETABLE_SHIFT - PAGE_SHIFT);
+ continue;
+ }
+ l2e = sh2_map_domain_page(_mfn(l3e_get_pfn(l3e[i3])));
+#endif /* all levels... */
+ for ( i2 = 0; i2 < L2_PAGETABLE_ENTRIES; i2++ )
+ {
+ if ( !(l2e_get_flags(l2e[i2]) & _PAGE_PRESENT) )
+ {
+ gfn += 1 << (L2_PAGETABLE_SHIFT - PAGE_SHIFT);
+ continue;
+ }
+ l1e = sh2_map_domain_page(_mfn(l2e_get_pfn(l2e[i2])));
+
+ for ( i1 = 0; i1 < L1_PAGETABLE_ENTRIES; i1++, gfn++ )
+ {
+ if ( !(l1e_get_flags(l1e[i1]) & _PAGE_PRESENT) )
+ continue;
+ mfn = l1e_get_pfn(l1e[i1]);
+ ASSERT(valid_mfn(_mfn(mfn)));
+ m2pfn = get_gpfn_from_mfn(mfn);
+ if ( m2pfn != gfn )
+ {
+ pmbad++;
+ SHADOW2_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
+ " -> gfn %#lx\n", gfn, mfn, m2pfn);
+ BUG();
+ }
+ }
+ sh2_unmap_domain_page(l1e);
+ }
+#if CONFIG_PAGING_LEVELS >= 3
+ sh2_unmap_domain_page(l2e);
+ }
+#if CONFIG_PAGING_LEVELS >= 4
+ sh2_unmap_domain_page(l3e);
+ }
+#endif
+#endif
+
+#if CONFIG_PAGING_LEVELS == 4
+ sh2_unmap_domain_page(l4e);
+#elif CONFIG_PAGING_LEVELS == 3
+ sh2_unmap_domain_page(l3e);
+#else /* CONFIG_PAGING_LEVELS == 2 */
+ sh2_unmap_domain_page(l2e);
+#endif
+
+ }
+
+ //SHADOW2_PRINTK("p2m audit complete\n");
+ //if ( orphans_i | orphans_d | mpbad | pmbad )
+ // SHADOW2_PRINTK("p2m audit found %lu orphans (%lu inval %lu debug)\n",
+ // orphans_i + orphans_d, orphans_i, orphans_d,
+ if ( mpbad | pmbad )
+ SHADOW2_PRINTK("p2m audit found %lu odd p2m, %lu bad m2p entries\n",
+ pmbad, mpbad);
+}
+
+#endif /* p2m audit */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-set-style: "BSD"
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff -r fda70200da01 -r 0f917d63e960 xen/arch/x86/shadow2.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/arch/x86/shadow2.c Wed Aug 16 17:02:35 2006 +0100
@@ -0,0 +1,4469 @@
+/******************************************************************************
+ * arch/x86/shadow2.c
+ *
+ * Simple, mostly-synchronous shadow page tables.
+ * Parts of this code are Copyright (c) 2006 by XenSource Inc.
+ * Parts of this code are Copyright (c) 2006 by Michael A Fetterman
+ * Parts based on earlier work by Michael A Fetterman, Ian Pratt et al.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+// DESIGN QUESTIONS:
+// Why use subshadows for PAE guests?
+// - reduces pressure in the hash table
+// - reduces shadow size (64-vs-4096 bytes of shadow for 32 bytes of guest L3)
+// - would need to find space in the page_info to store 7 more bits of
+// backpointer
+// - independent shadows of 32 byte chunks makes it non-obvious how to quickly
+// figure out when to demote the guest page from l3 status
+//
+// PAE Xen HVM guests are restricted to 8GB of pseudo-physical address space.
+// - Want to map the P2M table into the 16MB RO_MPT hole in Xen's address
+// space for both PV and HVM guests.
+//
+
+#define SHADOW2 1
+
+#include <xen/config.h>
+#include <xen/types.h>
+#include <xen/mm.h>
+#include <xen/trace.h>
+#include <xen/sched.h>
+#include <xen/perfc.h>
+#include <xen/domain_page.h>
+#include <asm/page.h>
+#include <asm/current.h>
+#include <asm/shadow2.h>
+#include <asm/shadow2-private.h>
+#include <asm/shadow2-types.h>
+#include <asm/flushtlb.h>
+#include <asm/hvm/hvm.h>
+
+/* The first cut: an absolutely synchronous, trap-and-emulate version,
+ * supporting only HVM guests (and so only "external" shadow mode).
+ *
+ * THINGS TO DO LATER:
+ *
+ * FIX GVA_TO_GPA
+ * The current interface returns an unsigned long, which is not big enough
+ * to hold a physical address in PAE. Should return a gfn instead.
+ *
+ * TEARDOWN HEURISTICS
+ * Also: have a heuristic for when to destroy a previous paging-mode's
+ * shadows. When a guest is done with its start-of-day 32-bit tables
+ * and reuses the memory we want to drop those shadows. Start with
+ * shadows in a page in two modes as a hint, but beware of clever tricks
+ * like reusing a pagetable for both PAE and 64-bit during boot...
+ *
+ * PAE LINEAR MAPS
+ * Rework shadow_get_l*e() to have the option of using map_domain_page()
+ * instead of linear maps. Add appropriate unmap_l*e calls in the users.
+ * Then we can test the speed difference made by linear maps. If the
+ * map_domain_page() version is OK on PAE, we could maybe allow a lightweight
+ * l3-and-l2h-only shadow mode for PAE PV guests that would allow them
+ * to share l2h pages again.
+ *
+ * PAE L3 COPYING
+ * In this code, we copy all 32 bytes of a PAE L3 every time we change an
+ * entry in it, and every time we change CR3. We copy it for the linear
+ * mappings (ugh! PAE linear mappings) and we copy it to the low-memory
+ * buffer so it fits in CR3. Maybe we can avoid some of this recopying
+ * by using the shadow directly in some places.
+ * Also, for SMP, need to actually respond to seeing shadow2_pae_flip_pending.
+ *
+ * GUEST_WALK_TABLES TLB FLUSH COALESCE
+ * guest_walk_tables can do up to three remote TLB flushes as it walks to
+ * the first l1 of a new pagetable. Should coalesce the flushes to the end,
+ * and if we do flush, re-do the walk. If anything has changed, then
+ * pause all the other vcpus and do the walk *again*.
+ *
+ * WP DISABLED
+ * Consider how to implement having the WP bit of CR0 set to 0.
+ * Since we need to be able to cause write faults to pagetables, this might
+ * end up looking like not having the (guest) pagetables present at all in
+ * HVM guests...
+ *
+ * PSE disabled / PSE36
+ * We don't support any modes other than PSE enabled, PSE36 disabled.
+ * Neither of those would be hard to change, but we'd need to be able to
+ * deal with shadows made in one mode and used in another.
+ */
+
+#define FETCH_TYPE_PREFETCH 1
+#define FETCH_TYPE_DEMAND 2
+#define FETCH_TYPE_WRITE 4
+typedef enum {
+ ft_prefetch = FETCH_TYPE_PREFETCH,
+ ft_demand_read = FETCH_TYPE_DEMAND,
+ ft_demand_write = FETCH_TYPE_DEMAND | FETCH_TYPE_WRITE,
+} fetch_type_t;
+
+#ifndef NDEBUG
+static char *fetch_type_names[] = {
+ [ft_prefetch] "prefetch",
+ [ft_demand_read] "demand read",
+ [ft_demand_write] "demand write",
+};
+#endif
+
+/* XXX forward declarations */
+#if (GUEST_PAGING_LEVELS == 3) && (SHADOW_PAGING_LEVELS == 3)
+static unsigned long hvm_pae_copy_root(struct vcpu *v, l3_pgentry_t *l3tab,
int clear_res);
+#endif
+static inline void sh2_update_linear_entries(struct vcpu *v);
+
+/**************************************************************************/
+/* Hash table mapping from guest pagetables to shadows
+ *
+ * Normal case: maps the mfn of a guest page to the mfn of its shadow page.
+ * FL1's: maps the *gfn* of the start of a superpage to the mfn of a
+ * shadow L1 which maps its "splinters".
+ * PAE CR3s: maps the 32-byte aligned, 32-bit CR3 value to the mfn of the
+ * PAE L3 info page for that CR3 value.
+ */
+
+static inline mfn_t
+get_fl1_shadow_status(struct vcpu *v, gfn_t gfn)
+/* Look for FL1 shadows in the hash table */
+{
+ mfn_t smfn = shadow2_hash_lookup(v, gfn_x(gfn),
+ PGC_SH2_fl1_shadow >> PGC_SH2_type_shift);
+
+ if ( unlikely(shadow2_mode_log_dirty(v->domain) && valid_mfn(smfn)) )
+ {
+ struct page_info *page = mfn_to_page(smfn);
+ if ( !(page->count_info & PGC_SH2_log_dirty) )
+ shadow2_convert_to_log_dirty(v, smfn);
+ }
+
+ return smfn;
+}
+
+static inline mfn_t
+get_shadow_status(struct vcpu *v, mfn_t gmfn, u32 shadow_type)
+/* Look for shadows in the hash table */
+{
+ mfn_t smfn = shadow2_hash_lookup(v, mfn_x(gmfn),
+ shadow_type >> PGC_SH2_type_shift);
+ perfc_incrc(shadow2_get_shadow_status);
+
+ if ( unlikely(shadow2_mode_log_dirty(v->domain) && valid_mfn(smfn)) )
+ {
+ struct page_info *page = mfn_to_page(smfn);
+ if ( !(page->count_info & PGC_SH2_log_dirty) )
+ shadow2_convert_to_log_dirty(v, smfn);
+ }
+
+ return smfn;
+}
+
+static inline void
+set_fl1_shadow_status(struct vcpu *v, gfn_t gfn, mfn_t smfn)
+/* Put an FL1 shadow into the hash table */
+{
+ SHADOW2_PRINTK("gfn=%"SH2_PRI_gfn", type=%08x, smfn=%05lx\n",
+ gfn_x(gfn), PGC_SH2_fl1_shadow, mfn_x(smfn));
+
+ if ( unlikely(shadow2_mode_log_dirty(v->domain)) )
_______________________________________________
Xen-changelog mailing list
Xen-changelog@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-changelog
|
Home