diff -r 388c44a63613 tools/libxc/xc_dom.h --- a/tools/libxc/xc_dom.h Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxc/xc_dom.h Tue Nov 16 14:48:46 2010 -0500 @@ -91,6 +91,8 @@ /* physical memory */ xen_pfn_t total_pages; + /* start of the pci_hole. goes up to 4gb */ + xen_pfn_t pci_hole; struct xc_dom_phys *phys_pages; int realmodearea_log; diff -r 388c44a63613 tools/libxc/xc_dom_core.c --- a/tools/libxc/xc_dom_core.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxc/xc_dom_core.c Tue Nov 16 14:48:46 2010 -0500 @@ -714,18 +714,22 @@ { uint32_t *p2m_32; uint64_t *p2m_64; - xen_pfn_t i; + xen_pfn_t i, tot_pages; if ( !dom->p2m_guest ) return 0; + tot_pages = dom->total_pages; + if (dom->pci_hole) + tot_pages += (0x100000 - dom->pci_hole); + switch ( dom->arch_hooks->sizeof_pfn ) { case 4: DOMPRINTF("%s: dst 32bit, pages 0x%" PRIpfn "", - __FUNCTION__, dom->total_pages); + __FUNCTION__, tot_pages); p2m_32 = dom->p2m_guest; - for ( i = 0; i < dom->total_pages; i++ ) + for ( i = 0; i < tot_pages; i++ ) if ( dom->p2m_host[i] != INVALID_P2M_ENTRY ) p2m_32[i] = dom->p2m_host[i]; else @@ -733,9 +737,9 @@ break; case 8: DOMPRINTF("%s: dst 64bit, pages 0x%" PRIpfn "", - __FUNCTION__, dom->total_pages); + __FUNCTION__, tot_pages); p2m_64 = dom->p2m_guest; - for ( i = 0; i < dom->total_pages; i++ ) + for ( i = 0; i < tot_pages; i++ ) if ( dom->p2m_host[i] != INVALID_P2M_ENTRY ) p2m_64[i] = dom->p2m_host[i]; else diff -r 388c44a63613 tools/libxc/xc_dom_x86.c --- a/tools/libxc/xc_dom_x86.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxc/xc_dom_x86.c Tue Nov 16 14:48:46 2010 -0500 @@ -406,6 +406,8 @@ { size_t p2m_size = dom->total_pages * dom->arch_hooks->sizeof_pfn; + if (dom->pci_hole && (dom->total_pages > dom->pci_hole)) + p2m_size += (0x100000 - dom->pci_hole) * dom->arch_hooks->sizeof_pfn; /* allocate phys2mach table */ if ( xc_dom_alloc_segment(dom, &dom->p2m_seg, "phys2mach", 0, p2m_size) ) return -1; @@ -712,6 +714,7 @@ { int rc; xen_pfn_t pfn, allocsz, i, j, mfn; + size_t p2m_size; rc = x86_compat(dom->xch, dom->guest_domid, dom->guest_type); if ( rc ) @@ -723,8 +726,13 @@ if ( rc ) return rc; } + p2m_size = dom->total_pages; - dom->p2m_host = xc_dom_malloc(dom, sizeof(xen_pfn_t) * dom->total_pages); + if (dom->pci_hole && (dom->total_pages > dom->pci_hole)) + p2m_size += (0x100000 - dom->pci_hole); + + DOMPRINTF("Allocating 0x%" PRIpfn " bytes for P2M.", (xen_pfn_t)(p2m_size * sizeof(xen_pfn_t))); + dom->p2m_host = xc_dom_malloc(dom, sizeof(xen_pfn_t) * p2m_size); if ( dom->superpages ) { int count = dom->total_pages >> SUPERPAGE_PFN_SHIFT; @@ -750,21 +758,66 @@ } else { - /* setup initial p2m */ - for ( pfn = 0; pfn < dom->total_pages; pfn++ ) - dom->p2m_host[pfn] = pfn; - - /* allocate guest memory */ - for ( i = rc = allocsz = 0; - (i < dom->total_pages) && !rc; - i += allocsz ) + /* for PCI mapping, stick INVALID_MFN in the PCI_HOLE */ + if ( dom->pci_hole && (dom->total_pages > dom->pci_hole) ) { - allocsz = dom->total_pages - i; - if ( allocsz > 1024*1024 ) - allocsz = 1024*1024; - rc = xc_domain_populate_physmap_exact( - dom->xch, dom->guest_domid, allocsz, - 0, 0, &dom->p2m_host[i]); + /* setup initial p2m in three passes. */ + for (pfn = 0; pfn < dom->pci_hole; pfn++) + dom->p2m_host[pfn] = pfn; + + xc_dom_printf (dom->xch, "%s: 0x0->0x%" PRIpfn " has PFNs.", __FUNCTION__, pfn); + xc_dom_printf (dom->xch, "%s: 0x%" PRIpfn " -> 0x10000 has INVALID_MFN", + __FUNCTION__, pfn); + for (; pfn < 0x100000; pfn++) + dom->p2m_host[pfn] = INVALID_MFN; + + for (; pfn < 0x100000 + dom->total_pages - dom->pci_hole; pfn++) + dom->p2m_host[pfn] = pfn; + xc_dom_printf (dom->xch, "%s: 0x100000 -> 0x%" PRIpfn " has PFNs.", __FUNCTION__, + pfn); + + /* allocate guest memory in two passes. */ + for (i = rc = allocsz = 0; (i < dom->pci_hole) && !rc; i += allocsz) + { + allocsz = dom->pci_hole - i; + xc_dom_printf (dom->xch, "%s: Populating M2P 0x%" PRIpfn "->0x%" PRIpfn, + __FUNCTION__, i, i + allocsz); + rc = xc_domain_populate_physmap_exact (dom->xch, dom->guest_domid, + allocsz, 0, 0, + &dom->p2m_host[i]); + } + for (i = 0x100000, allocsz = rc = 0; + (i < (0x100000 + dom->total_pages - dom->pci_hole)) + && !rc; i += allocsz) + { + allocsz = (dom->total_pages - dom->pci_hole) - (i - 0x100000); + if (allocsz > 1024 * 1024) + allocsz = 1024 * 1024; + xc_dom_printf (dom->xch, "%s: Populating M2P 0x%" PRIpfn "->0x%" PRIpfn, + __FUNCTION__, i, i + allocsz); + rc = xc_domain_populate_physmap_exact (dom->xch, dom->guest_domid, + allocsz, 0, 0, + &dom->p2m_host[i]); + } + xc_dom_printf (dom->xch, "%s: Done with PCI populate physmap", + __FUNCTION__); + } else { + /* setup initial p2m */ + for ( pfn = 0; pfn < dom->total_pages; pfn++ ) + dom->p2m_host[pfn] = pfn; + + /* allocate guest memory */ + for ( i = rc = allocsz = 0; + (i < dom->total_pages) && !rc; + i += allocsz ) + { + allocsz = dom->total_pages - i; + if ( allocsz > 1024*1024 ) + allocsz = 1024*1024; + rc = xc_domain_populate_physmap_exact( + dom->xch, dom->guest_domid, allocsz, + 0, 0, &dom->p2m_host[i]); + } } } diff -r 388c44a63613 tools/libxc/xc_domain.c --- a/tools/libxc/xc_domain.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxc/xc_domain.c Tue Nov 16 14:48:46 2010 -0500 @@ -481,16 +481,25 @@ #include "xc_e820.h" int xc_domain_set_memmap_limit(xc_interface *xch, uint32_t domid, - unsigned long map_limitkb) + unsigned long map_limitkb, + xen_pfn_t pci_hole_start) { int rc; + uint64_t delta_kb; + size_t e820_sz; struct xen_foreign_memory_map fmap = { .domid = domid, .map = { .nr_entries = 1 } }; DECLARE_HYPERCALL_BUFFER(struct e820entry, e820); - e820 = xc_hypercall_buffer_alloc(xch, e820, sizeof(*e820)); + delta_kb = map_limitkb - (uint64_t)(pci_hole_start << 2); + if (pci_hole_start && (delta_kb > 0)) + e820_sz = sizeof(*e820); + else + e820_sz = sizeof(*e820)*2; + + e820 = xc_hypercall_buffer_alloc(xch, e820, e820_sz); if ( e820 == NULL ) { @@ -502,6 +511,16 @@ e820->size = (uint64_t)map_limitkb << 10; e820->type = E820_RAM; + if (pci_hole_start && (delta_kb > 0)) + { + fmap.map.nr_entries ++; + e820[0].size = (uint64_t)pci_hole_start << 12; + + e820[1].type = E820_RAM; + e820[1].addr = (uint64_t)0x100000 << 12; /* val in pfn... */ + e820[1].size = (uint64_t)delta_kb << 10; /* .. while here in in kB. */ + } + set_xen_guest_handle(fmap.map.buffer, e820); rc = do_memory_op(xch, XENMEM_set_memory_map, &fmap, sizeof(fmap)); @@ -513,7 +532,8 @@ #else int xc_domain_set_memmap_limit(xc_interface *xch, uint32_t domid, - unsigned long map_limitkb) + unsigned long map_limitkb, + xen_pfn_t pci_hole_start) { PERROR("Function not implemented"); errno = ENOSYS; diff -r 388c44a63613 tools/libxc/xenctrl.h --- a/tools/libxc/xenctrl.h Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxc/xenctrl.h Tue Nov 16 14:48:46 2010 -0500 @@ -913,7 +913,8 @@ int xc_domain_set_memmap_limit(xc_interface *xch, uint32_t domid, - unsigned long map_limitkb); + unsigned long map_limitkb, + xen_pfn_t pci_hole_start); int xc_domain_set_time_offset(xc_interface *xch, uint32_t domid, diff -r 388c44a63613 tools/libxl/libxl.h --- a/tools/libxl/libxl.h Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxl/libxl.h Tue Nov 16 14:48:46 2010 -0500 @@ -392,6 +392,7 @@ libxl_device_disk *disk, libxl_diskinfo *diskinfo); int libxl_cdrom_insert(libxl_ctx *ctx, uint32_t domid, libxl_device_disk *disk); +int libxl_find_pci_hole(uint32_t *start_pfn); /* * Make a disk available in this domain. Returns path to a device. */ diff -r 388c44a63613 tools/libxl/libxl.idl --- a/tools/libxl/libxl.idl Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxl/libxl.idl Tue Nov 16 14:48:46 2010 -0500 @@ -110,6 +110,7 @@ ])), ("pv", "!%s", Struct(None, [("slack_memkb", uint32), + ("pci_hole_start", uint32), ("bootloader", string), ("bootloader_args", string), ("cmdline", string), diff -r 388c44a63613 tools/libxl/libxl_dom.c --- a/tools/libxl/libxl_dom.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxl/libxl_dom.c Tue Nov 16 14:48:46 2010 -0500 @@ -71,7 +71,8 @@ xc_domain_setmaxmem(ctx->xch, domid, info->target_memkb + LIBXL_MAXMEM_CONSTANT); xc_domain_set_memmap_limit(ctx->xch, domid, (info->hvm) ? info->max_memkb : - (info->max_memkb + info->u.pv.slack_memkb)); + (info->max_memkb + info->u.pv.slack_memkb), + (info->hvm) ? 0 : info->u.pv.pci_hole_start); xc_domain_set_tsc_info(ctx->xch, domid, info->tsc_mode, 0, 0, 0); if ( info->disable_migrate ) xc_domain_disable_migrate(ctx->xch, domid); @@ -181,6 +182,8 @@ } } } + if ( info->u.pv.pci_hole_start) + dom->pci_hole = info->u.pv.pci_hole_start; dom->flags = flags; dom->console_evtchn = state->console_port; diff -r 388c44a63613 tools/libxl/libxl_pci.c --- a/tools/libxl/libxl_pci.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxl/libxl_pci.c Tue Nov 16 14:48:46 2010 -0500 @@ -1066,3 +1066,51 @@ free(pcidevs); return 0; } + +#define MAX_LINE 300 +int libxl_find_pci_hole(uint32_t *start_pfn) +{ + FILE *fp; + char *s; + char buf[MAX_LINE]; + int ret = -ENODEV; + long int pci_hole_phys; + + *start_pfn = 0; + fp = fopen("/proc/iomem", "r"); + if (!fp) + return ret; + + while (1) { + s = fgets(buf, MAX_LINE, fp); + if (!s) + break; + if (strlen(buf) < 1) + continue; + if (buf[strlen(buf)-1] == '\n') + buf[strlen(buf)-1] = '\0'; + s = strchr(buf,'P'); + if (!s) + continue; + if (strncmp(s, "PCI", 3) == 0) { + if (buf[0] == ' ') + continue; + s = strchr(buf,'-'); + if (!s) + break; + s[0]='\0'; + pci_hole_phys = strtol(buf, NULL, 16); + if (!pci_hole_phys) + break; + /* We don't want to the holes below 16MB. */ + if (pci_hole_phys <= 0x1000) + continue; + *start_pfn = pci_hole_phys >> 12; + fprintf(stderr,"The value is 0x%d\n", *start_pfn); + ret = 0; + break; + } + } + fclose(fp); + return ret; +} diff -r 388c44a63613 tools/libxl/xl_cmdimpl.c --- a/tools/libxl/xl_cmdimpl.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/libxl/xl_cmdimpl.c Tue Nov 16 14:48:46 2010 -0500 @@ -1078,6 +1078,14 @@ if (!xlu_cfg_get_long (config, "pci_power_mgmt", &l)) pci_power_mgmt = l; + if (!xlu_cfg_get_long (config, "pci_hole", &l)) { + if (l == 1) { + uint32_t pfn_start = 0; + if (!libxl_find_pci_hole(&pfn_start)) + b_info->u.pv.pci_hole_start = pfn_start; + } else + b_info->u.pv.pci_hole_start = l; + } if (!xlu_cfg_get_list (config, "pci", &pcis, 0, 0)) { int i; d_config->num_pcidevs = 0; diff -r 388c44a63613 tools/python/xen/lowlevel/xc/xc.c --- a/tools/python/xen/lowlevel/xc/xc.c Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/python/xen/lowlevel/xc/xc.c Tue Nov 16 14:48:46 2010 -0500 @@ -458,6 +458,7 @@ unsigned int mem_mb; unsigned long store_mfn = 0; unsigned long console_mfn = 0; + unsigned long pci_hole_start = 0; PyObject* elfnote_dict; PyObject* elfnote = NULL; PyObject* ret; @@ -467,14 +468,16 @@ "console_evtchn", "image", /* optional */ "ramdisk", "cmdline", "flags", - "features", "vhpt", "superpages", NULL }; - - if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iiiis|ssisii", kwd_list, + "features", "vhpt", "superpages", + "pci_hole", NULL }; + + if ( !PyArg_ParseTupleAndKeywords(args, kwds, "iiiis|ssisiii", kwd_list, &domid, &store_evtchn, &mem_mb, &console_evtchn, &image, /* optional */ &ramdisk, &cmdline, &flags, - &features, &vhpt, &superpages) ) + &features, &vhpt, &superpages, + &pci_hole_start) ) return NULL; xc_dom_loginit(self->xc_handle); @@ -486,6 +489,8 @@ dom->superpages = superpages; + dom->pci_hole = pci_hole_start; + if ( xc_dom_linux_build(self->xc_handle, dom, domid, mem_mb, image, ramdisk, flags, store_evtchn, &store_mfn, console_evtchn, &console_mfn) != 0 ) { @@ -1659,11 +1664,13 @@ { uint32_t dom; unsigned int maplimit_kb; - - if ( !PyArg_ParseTuple(args, "ii", &dom, &maplimit_kb) ) + unsigned long pci_hole_start = 0; + + if ( !PyArg_ParseTuple(args, "ii|i", &dom, &maplimit_kb, &pci_hole_start) ) return NULL; - if ( xc_domain_set_memmap_limit(self->xc_handle, dom, maplimit_kb) != 0 ) + if ( xc_domain_set_memmap_limit(self->xc_handle, dom, maplimit_kb, + pci_hole_start) != 0 ) return pyxc_error_to_exception(self->xc_handle); Py_INCREF(zero); @@ -2661,6 +2668,7 @@ "Set a domain's physical memory mappping limit\n" " dom [int]: Identifier of domain.\n" " map_limitkb [int]: .\n" + " pci_hole_start [int]: PFN for start of PCI hole (optional).\n" "Returns: [int] 0 on success; -1 on error.\n" }, #ifdef __ia64__ diff -r 388c44a63613 tools/python/xen/xend/XendConfig.py --- a/tools/python/xen/xend/XendConfig.py Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/python/xen/xend/XendConfig.py Tue Nov 16 14:48:46 2010 -0500 @@ -241,6 +241,7 @@ 'suppress_spurious_page_faults': bool0, 's3_integrity' : int, 'superpages' : int, + 'pci_hole' : int, 'memory_sharing': int, 'pool_name' : str, 'Description': str, @@ -422,6 +423,7 @@ 'target': 0, 'pool_name' : 'Pool-0', 'superpages': 0, + 'pci_hole': 0, 'description': '', } @@ -2135,6 +2137,9 @@ image.append(['args', self['PV_args']]) if self.has_key('superpages'): image.append(['superpages', self['superpages']]) + if self.has_key('pci_hole'): + image.append(['pci_hole', self['pci_hole']]) + for key in XENAPI_PLATFORM_CFG_TYPES.keys(): if key in self['platform']: @@ -2179,6 +2184,10 @@ val = sxp.child_value(image_sxp, 'superpages') if val is not None: self['superpages'] = val + + val = sxp.child_value(image_sxp, 'pci_hole') + if val is not None: + self['pci_hole'] = val val = sxp.child_value(image_sxp, 'memory_sharing') if val is not None: diff -r 388c44a63613 tools/python/xen/xend/image.py --- a/tools/python/xen/xend/image.py Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/python/xen/xend/image.py Tue Nov 16 14:48:46 2010 -0500 @@ -84,6 +84,7 @@ ostype = None superpages = 0 + pci_hole = 0 memory_sharing = 0 def __init__(self, vm, vmConfig): @@ -711,6 +712,7 @@ self.vramsize = int(vmConfig['platform'].get('videoram',4)) * 1024 self.is_stubdom = (self.kernel.find('stubdom') >= 0) self.superpages = int(vmConfig['superpages']) + self.pci_hole = int(vmConfig['pci_hole']) def buildDomain(self): store_evtchn = self.vm.getStorePort() @@ -729,6 +731,7 @@ log.debug("features = %s", self.vm.getFeatures()) log.debug("flags = %d", self.flags) log.debug("superpages = %d", self.superpages) + log.debug("pci_hole = %d", self.pci_hole) if arch.type == "ia64": log.debug("vhpt = %d", self.vhpt) @@ -742,7 +745,8 @@ features = self.vm.getFeatures(), flags = self.flags, vhpt = self.vhpt, - superpages = self.superpages) + superpages = self.superpages, + pci_hole = self.pci_hole) def getBitSize(self): return xc.getBitSize(image = self.kernel, @@ -774,7 +778,6 @@ args = args + ([ "-M", "xenpv"]) return args - class HVMImageHandler(ImageHandler): ostype = "hvm" @@ -1065,7 +1068,7 @@ # set physical mapping limit # add an 8MB slack to balance backend allocations. mem_kb = self.getRequiredMaximumReservation() + (8 * 1024) - xc.domain_set_memmap_limit(self.vm.getDomid(), mem_kb) + xc.domain_set_memmap_limit(self.vm.getDomid(), mem_kb, self.pci_hole) rc = LinuxImageHandler.buildDomain(self) self.setCpuid() return rc diff -r 388c44a63613 tools/python/xen/xm/create.py --- a/tools/python/xen/xm/create.py Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/python/xen/xm/create.py Tue Nov 16 14:48:46 2010 -0500 @@ -680,6 +680,11 @@ fn=set_int, default=0, use="Create domain with superpages") +gopts.var('pci_hole', val='0x|0', + fn=set_int, default=0, + use="""Create domain with a PCI hole. The value is the PFN of the + start of PCI hole. Usually that is 0xc0000.""") + def err(msg): """Print an error to stderr and exit. """ @@ -770,6 +775,9 @@ config_image.append(['args', vals.extra]) if vals.superpages: config_image.append(['superpages', vals.superpages]) + if vals.pci_hole: + config_image.append(['pci_hole', vals.pci_hole]) + if vals.builder == 'hvm': configure_hvm(config_image, vals) diff -r 388c44a63613 tools/python/xen/xm/xenapi_create.py --- a/tools/python/xen/xm/xenapi_create.py Wed Nov 10 14:56:06 2010 +0000 +++ b/tools/python/xen/xm/xenapi_create.py Tue Nov 16 14:48:46 2010 -0500 @@ -285,6 +285,8 @@ vm.attributes["s3_integrity"].value, "superpages": vm.attributes["superpages"].value, + "pci_hole": + vm.attributes["pci_hole"].value, "memory_static_max": get_child_node_attribute(vm, "memory", "static_max"), "memory_static_min": @@ -697,6 +699,8 @@ = str(get_child_by_name(config, "s3_integrity", 0)) vm.attributes["superpages"] \ = str(get_child_by_name(config, "superpages", 0)) + vm.attributes["pci_hole"] \ + = str(get_child_by_name(config, "pci_hole", 0)) vm.attributes["pool_name"] \ = str(get_child_by_name(config, "pool_name", "Pool-0"))