* Ian Pratt <m+Ian.Pratt@xxxxxxxxxxxx> [2005-04-14 10:50]:
> > The following patch updates the dom0 pincpu operation to read
> > the VCPU value from the xend interface rather than
> > hard-coding the exec_domain to 0. This prevented pinning
> > VCPUS other than 0 to a particular cpu. I added the number
> > of VCPUS to the main xm list output and also included a new
> > sub-option to xm list to display the VCPU to CPU mapping.
> > While working on the pincpu code, I fixed an out-of-bounds
> > indexing for the pincpu operation that wasn't previously
> > exposed since the vcpu/exec_domain value was hard-coded to 0.
>
> Ryan, good progress, but I'd like to propose a couple of extentions:
>
> It would be useful if you could update it so that pincpu enabled you to
> specify a set of physical CPUs for each VCPU e.g.
>
> "xm pincpu mydom 1 2,4-6" which would allow VCPU 1 of mydom to run on
> CPUs 2,4 and 5 but no others. -1 would still mean "run anywhere". Having
> this functionality is really important before we can implement any kind
> of CPU load ballancer.
Attached is a patch that depends on the previous [1]patch, replacing the
pincpu cpu argument with cpumap, a u32 bitmap representing which CPUs a
VCPU can use.
xm pincpu now takes a comma separated list to describe the bitmap, just
as you listed in your example.
(hungerforce) root # xm pincpu debian_sarge_2 1 2,4-6
(hungerforce) root # xm list
Name Id Mem(MB) CPU VCPU(s) State Time(s) Console
Domain-0 0 507 0 2 r---- 12.8
debian_sarge_2 1 128 1 4 -b--- 2.2 9601
(hungerforce) root # xm list -v debian_sarge_2
Name Id VCPU CPU CPUMAP
debian_sarge_2 1 0 1 0xffffffff
debian_sarge_2 1 1 0 0x00000074
debian_sarge_2 1 2 1 0xffffffff
debian_sarge_2 1 3 0 0xffffffff
The hypervisor now updates the exec_domain's cpumap value, and then, for
now, just picks the first set bit from the new cpumap passed. So, for
folks who want to just one to one pinning, one can still do things like:
xm pincpu mydom 1 0 which pins vcpu 1 in mydom to cpu 0.
Please apply.
1. http://lists.xensource.com/archives/html/xen-devel/2005-04/msg00412.html
--
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253 T/L: 678-9253
ryanh@xxxxxxxxxx
diffstat output:
tools/libxc/xc.h | 3 +-
tools/libxc/xc_domain.c | 6 ++--
tools/python/xen/lowlevel/xc/xc.c | 27 +++++++++++-------
tools/python/xen/xend/XendClient.py | 4 +-
tools/python/xen/xend/XendDomain.py | 8 ++---
tools/python/xen/xend/XendDomainInfo.py | 3 +-
tools/python/xen/xend/server/SrvDomain.py | 2 -
tools/python/xen/xm/main.py | 43 ++++++++++++++++++++++--------
xen/arch/x86/domain.c | 1
xen/common/dom0_ops.c | 18 +++++++++---
xen/common/domain.c | 1
xen/include/public/dom0_ops.h | 7 ++--
xen/include/xen/sched.h | 4 ++
13 files changed, 87 insertions(+), 40 deletions(-)
Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
---
diff -urN b/tools/libxc/xc_domain.c c/tools/libxc/xc_domain.c
--- b/tools/libxc/xc_domain.c 2005-04-15 11:05:54.000000000 -0500
+++ c/tools/libxc/xc_domain.c 2005-04-15 14:37:16.000000000 -0500
@@ -65,13 +65,13 @@
int xc_domain_pincpu(int xc_handle,
u32 domid,
int vcpu,
- int cpu)
+ u32 cpumap)
{
dom0_op_t op;
op.cmd = DOM0_PINCPUDOMAIN;
op.u.pincpudomain.domain = (domid_t)domid;
op.u.pincpudomain.exec_domain = vcpu;
- op.u.pincpudomain.cpu = cpu;
+ op.u.pincpudomain.cpumap = cpumap;
return do_dom0_op(xc_handle, &op);
}
@@ -116,6 +116,8 @@
info->vcpus = op.u.getdomaininfo.n_vcpu;
memcpy(info->vcpu_to_cpu, &op.u.getdomaininfo.vcpu_to_cpu,
MAX_VIRT_CPUS*sizeof(u32));
+ memcpy(info->cpumap, &op.u.getdomaininfo.cpumap,
+ MAX_VIRT_CPUS*sizeof(u32));
next_domid = (u16)op.u.getdomaininfo.domain + 1;
info++;
diff -urN b/tools/libxc/xc.h c/tools/libxc/xc.h
--- b/tools/libxc/xc.h 2005-04-15 11:05:54.000000000 -0500
+++ c/tools/libxc/xc.h 2005-04-15 14:18:44.000000000 -0500
@@ -87,6 +87,7 @@
u64 cpu_time;
unsigned long max_memkb;
u32 vcpu_to_cpu[MAX_VIRT_CPUS];
+ u32 cpumap[MAX_VIRT_CPUS];
} xc_dominfo_t;
typedef dom0_getdomaininfo_t xc_domaininfo_t;
@@ -131,7 +132,7 @@
int xc_domain_pincpu(int xc_handle,
u32 domid,
int vcpu,
- int cpu);
+ u32 cpumap);
/**
* This function will return information about one or more domains.
*
diff -urN b/tools/python/xen/lowlevel/xc/xc.c
c/tools/python/xen/lowlevel/xc/xc.c
--- b/tools/python/xen/lowlevel/xc/xc.c 2005-04-15 11:05:54.000000000 -0500
+++ c/tools/python/xen/lowlevel/xc/xc.c 2005-04-15 14:38:28.000000000 -0500
@@ -129,15 +129,15 @@
u32 dom;
int vcpu = 0;
- int cpu = -1;
+ unsigned long cpumap = 0xFFFFFFFF;
- static char *kwd_list[] = { "dom", "vcpu", "cpu", NULL };
+ static char *kwd_list[] = { "dom", "vcpu", "cpumap", NULL };
if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|ii", kwd_list,
- &dom, &vcpu, &cpu) )
+ &dom, &vcpu, &cpumap) )
return NULL;
- if ( xc_domain_pincpu(xc->xc_handle, dom, vcpu, cpu) != 0 )
+ if ( xc_domain_pincpu(xc->xc_handle, dom, vcpu, cpumap) != 0 )
return PyErr_SetFromErrno(xc_error);
Py_INCREF(zero);
@@ -149,7 +149,7 @@
PyObject *kwds)
{
XcObject *xc = (XcObject *)self;
- PyObject *list, *vcpu_list, *info_dict;
+ PyObject *list, *vcpu_list, *cpumap_list, *info_dict;
u32 first_dom = 0;
int max_doms = 1024, nr_doms, i, j;
@@ -170,8 +170,13 @@
for ( i = 0 ; i < nr_doms; i++ )
{
vcpu_list = PyList_New(MAX_VIRT_CPUS);
- for ( j = 0; j < MAX_VIRT_CPUS; j++ )
- PyList_SetItem( vcpu_list, j, Py_BuildValue("i",
info[i].vcpu_to_cpu[j]));
+ cpumap_list = PyList_New(MAX_VIRT_CPUS);
+ for ( j = 0; j < MAX_VIRT_CPUS; j++ ) {
+ PyList_SetItem( vcpu_list, j,
+ Py_BuildValue("i", info[i].vcpu_to_cpu[j]));
+ PyList_SetItem( cpumap_list, j,
+ Py_BuildValue("i", info[i].cpumap[j]));
+ }
info_dict = Py_BuildValue("{s:i,s:i,s:i,s:i,s:i,s:i,s:i,s:i,s:i"
",s:l,s:L,s:l,s:i}",
@@ -189,6 +194,7 @@
"maxmem_kb", info[i].max_memkb,
"shutdown_reason", info[i].shutdown_reason);
PyDict_SetItemString( info_dict, "vcpu_to_cpu", vcpu_list );
+ PyDict_SetItemString( info_dict, "cpumap", cpumap_list );
PyList_SetItem( list, i, info_dict);
}
@@ -884,9 +890,10 @@
{ "domain_pincpu",
(PyCFunction)pyxc_domain_pincpu,
METH_VARARGS | METH_KEYWORDS, "\n"
- "Pin a domain to a specified CPU.\n"
- " dom [int]: Identifier of domain to be pinned.\n"
- " cpu [int, -1]: CPU to pin to, or -1 to unpin\n\n"
+ "Pin a VCPU to a specified set CPUs.\n"
+ " dom [int]: Identifier of domain to which VCPU belongs.\n"
+ " vcpu [int, 0]: VCPU being pinned.\n"
+ " cpumap [int, -1]: Bitmap of usable CPUs.\n\n"
"Returns: [int] 0 on success; -1 on error.\n" },
{ "domain_getinfo",
diff -urN b/tools/python/xen/xend/server/SrvDomain.py
c/tools/python/xen/xend/server/SrvDomain.py
--- b/tools/python/xen/xend/server/SrvDomain.py 2005-04-15 11:05:54.000000000
-0500
+++ c/tools/python/xen/xend/server/SrvDomain.py 2005-04-15 14:32:58.000000000
-0500
@@ -93,7 +93,7 @@
fn = FormFn(self.xd.domain_pincpu,
[['dom', 'str'],
['vcpu', 'int'],
- ['cpu', 'int']])
+ ['cpumap', 'int']])
val = fn(req.args, {'dom': self.dom.id})
return val
diff -urN b/tools/python/xen/xend/XendClient.py
c/tools/python/xen/xend/XendClient.py
--- b/tools/python/xen/xend/XendClient.py 2005-04-15 11:05:54.000000000
-0500
+++ c/tools/python/xen/xend/XendClient.py 2005-04-15 11:31:05.000000000
-0500
@@ -246,11 +246,11 @@
'live' : live,
'resource' : resource })
- def xend_domain_pincpu(self, id, vcpu, cpu):
+ def xend_domain_pincpu(self, id, vcpu, cpumap):
return self.xendPost(self.domainurl(id),
{'op' : 'pincpu',
'vcpu' : vcpu,
- 'cpu' : cpu })
+ 'cpumap' : cpumap })
def xend_domain_cpu_bvt_set(self, id, mcuadv, warpback, warpvalue, warpl,
warpu):
return self.xendPost(self.domainurl(id),
diff -urN b/tools/python/xen/xend/XendDomainInfo.py
c/tools/python/xen/xend/XendDomainInfo.py
--- b/tools/python/xen/xend/XendDomainInfo.py 2005-04-15 11:05:54.000000000
-0500
+++ c/tools/python/xen/xend/XendDomainInfo.py 2005-04-15 11:35:05.000000000
-0500
@@ -373,6 +373,7 @@
sxpr.append(['cpu', self.info['cpu']])
sxpr.append(['cpu_time', self.info['cpu_time']/1e9])
sxpr.append(['vcpus', self.info['vcpus']])
+ sxpr.append(['cpumap', self.info['cpumap']])
sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x),
self.info['vcpu_to_cpu'][0:self.info['vcpus']]))])
@@ -452,7 +453,7 @@
raise VmError('missing memory size')
cpu = sxp.child_value(config, 'cpu')
if self.recreate and self.dom and cpu is not None:
- xc.domain_pincpu(self.dom, int(cpu))
+ xc.domain_pincpu(self.dom, 0, 1<<int(cpu))
try:
image = sxp.child_value(self.config, 'image')
self.vcpus = int(sxp.child_value(image, 'vcpus'))
diff -urN b/tools/python/xen/xend/XendDomain.py
c/tools/python/xen/xend/XendDomain.py
--- b/tools/python/xen/xend/XendDomain.py 2005-04-15 11:05:54.000000000
-0500
+++ c/tools/python/xen/xend/XendDomain.py 2005-04-15 11:33:21.000000000
-0500
@@ -610,16 +610,16 @@
xmigrate = XendMigrate.instance()
return xmigrate.save_begin(dominfo, dst)
- def domain_pincpu(self, id, vcpu, cpu):
- """Pin a vcpu in a domain to a cpu.
+ def domain_pincpu(self, id, vcpu, cpumap):
+ """Set which cpus vcpu can use
@param id: domain
@param vcpu: vcpu number
- @param cpu: cpu number
+ @param cpumap: bitmap of usbale cpus
"""
dominfo = self.domain_lookup(id)
try:
- return xc.domain_pincpu(int(dominfo.id), vcpu, cpu)
+ return xc.domain_pincpu(int(dominfo.id), vcpu, cpumap)
except Exception, ex:
raise XendError(str(ex))
diff -urN b/tools/python/xen/xm/main.py c/tools/python/xen/xm/main.py
--- b/tools/python/xen/xm/main.py 2005-04-15 11:05:54.000000000 -0500
+++ c/tools/python/xen/xm/main.py 2005-04-15 15:16:19.137184088 -0500
@@ -6,6 +6,8 @@
import sys
from getopt import getopt
import socket
+import warnings
+warnings.filterwarnings('ignore', category=FutureWarning)
from xen.xend import PrettyPrint
from xen.xend import sxp
@@ -401,19 +403,21 @@
% d)
def show_vcpus(self, doms):
- print 'Name Id VCPU CPU'
+ print 'Name Id VCPU CPU CPUMAP'
for dom in doms:
info = server.xend_domain(dom)
vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu',
'?').replace('-','')
+ cpumap = sxp.child_value(info, 'cpumap', [])
count = 0
for cpu in vcpu_to_cpu:
d = {}
- d['name'] = sxp.child_value(info, 'name', '??')
- d['dom'] = int(sxp.child_value(info, 'id', '-1'))
- d['vcpu'] = int(count)
- d['cpu'] = int(cpu)
+ d['name'] = sxp.child_value(info, 'name', '??')
+ d['dom'] = int(sxp.child_value(info, 'id', '-1'))
+ d['vcpu'] = int(count)
+ d['cpu'] = int(cpu)
+ d['cpumap'] = int(cpumap[count])
count = count + 1
- print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d" % d)
+ print ("%(name)-16s %(dom)3d %(vcpu)4d %(cpu)3d
0x%(cpumap)08x" % d)
def long_list(self, doms):
for dom in doms:
@@ -496,18 +500,35 @@
class ProgPincpu(Prog):
group = 'domain'
name = "pincpu"
- info = """Pin a vcpu to a cpu. """
+ info = """Set which cpus a VCPU can use. """
def help(self, args):
- print args[0],'DOM VCPU CPU'
- print '\nPin vcpu VCPU in domain DOM to cpu CPU.'
+ print args[0],'DOM VCPU CPUS'
+ print '\nSet which cpus VCPU in domain DOM can use.'
+
+ # convert list of cpus to bitmap integer value
+ def make_map(self, cpulist):
+ cpus = []
+ cpumap = 0
+ for c in cpulist.split(','):
+ if len(c) > 1:
+ (x,y) = c.split('-')
+ for i in range(int(x),int(y)+1):
+ cpus.append(int(i))
+ else:
+ cpus.append(int(c))
+ cpus.sort()
+ for c in cpus:
+ cpumap = cpumap | 1<<c
+
+ return cpumap
def main(self, args):
if len(args) != 4: self.err("%s: Invalid argument(s)" % args[0])
dom = args[1]
vcpu = int(args[2])
- cpu = int(args[3])
- server.xend_domain_pincpu(dom, vcpu, cpu)
+ cpumap = self.make_map(args[3]);
+ server.xend_domain_pincpu(dom, vcpu, cpumap)
xm.prog(ProgPincpu)
diff -urN b/xen/arch/x86/domain.c c/xen/arch/x86/domain.c
--- b/xen/arch/x86/domain.c 2005-04-14 22:08:58.000000000 -0500
+++ c/xen/arch/x86/domain.c 2005-04-15 13:38:08.000000000 -0500
@@ -240,6 +240,7 @@
d->shared_info = (void *)alloc_xenheap_page();
memset(d->shared_info, 0, PAGE_SIZE);
ed->vcpu_info = &d->shared_info->vcpu_data[ed->eid];
+ ed->cpumap = CPUMAP_RUNANYWHERE;
SHARE_PFN_WITH_DOMAIN(virt_to_page(d->shared_info), d);
machine_to_phys_mapping[virt_to_phys(d->shared_info) >>
PAGE_SHIFT] = INVALID_M2P_ENTRY;
diff -urN b/xen/common/dom0_ops.c c/xen/common/dom0_ops.c
--- b/xen/common/dom0_ops.c 2005-04-15 11:05:54.000000000 -0500
+++ c/xen/common/dom0_ops.c 2005-04-15 14:17:01.000000000 -0500
@@ -234,7 +234,7 @@
domid_t dom = op->u.pincpudomain.domain;
struct domain *d = find_domain_by_id(dom);
struct exec_domain *ed;
- int cpu = op->u.pincpudomain.cpu;
+ u32 cpumap = op->u.pincpudomain.cpumap;
if ( d == NULL )
{
@@ -264,17 +264,23 @@
break;
}
- if ( cpu == -1 )
+ /* update cpumap for this ed */
+ ed->cpumap = cpumap;
+
+ if ( cpumap == CPUMAP_RUNANYWHERE )
{
clear_bit(EDF_CPUPINNED, &ed->ed_flags);
}
else
{
+ /* pick a new cpu from the usable map */
+ int new_cpu = (int)find_first_set_bit(cpumap) % smp_num_cpus;
+
exec_domain_pause(ed);
- if ( ed->processor != (cpu % smp_num_cpus) )
+ if ( ed->processor != new_cpu )
set_bit(EDF_MIGRATED, &ed->ed_flags);
set_bit(EDF_CPUPINNED, &ed->ed_flags);
- ed->processor = cpu % smp_num_cpus;
+ ed->processor = new_cpu;
exec_domain_unpause(ed);
}
@@ -329,8 +335,10 @@
}
memset(&op->u.getdomaininfo.vcpu_to_cpu,-1,MAX_VIRT_CPUS*sizeof(u8));
- for_each_exec_domain ( d, ed )
+ for_each_exec_domain ( d, ed ) {
op->u.getdomaininfo.vcpu_to_cpu[ed->eid] = ed->processor;
+ op->u.getdomaininfo.cpumap[ed->eid] = ed->cpumap;
+ }
ed = d->exec_domain[op->u.getdomaininfo.exec_domain];
diff -urN b/xen/common/domain.c c/xen/common/domain.c
--- b/xen/common/domain.c 2005-04-14 22:08:58.000000000 -0500
+++ c/xen/common/domain.c 2005-04-15 15:00:02.000000000 -0500
@@ -320,6 +320,7 @@
ed = d->exec_domain[vcpu];
atomic_set(&ed->pausecnt, 0);
+ ed->cpumap = CPUMAP_RUNANYWHERE;
memcpy(&ed->arch, &idle0_exec_domain.arch, sizeof(ed->arch));
diff -urN b/xen/include/public/dom0_ops.h c/xen/include/public/dom0_ops.h
--- b/xen/include/public/dom0_ops.h 2005-04-15 11:05:54.000000000 -0500
+++ c/xen/include/public/dom0_ops.h 2005-04-15 14:14:48.000000000 -0500
@@ -93,7 +93,8 @@
memory_t shared_info_frame; /* MFN of shared_info struct */
u64 cpu_time;
u32 n_vcpu;
- u32 vcpu_to_cpu[MAX_VIRT_CPUS];
+ u32 vcpu_to_cpu[MAX_VIRT_CPUS]; /* current mapping */
+ u32 cpumap[MAX_VIRT_CPUS]; /* allowable mapping */
} dom0_getdomaininfo_t;
#define DOM0_SETDOMAININFO 13
@@ -176,14 +177,14 @@
} dom0_readconsole_t;
/*
- * Pin Domain to a particular CPU (use -1 to unpin)
+ * Set which cpus an exec_domain can use
*/
#define DOM0_PINCPUDOMAIN 20
typedef struct {
/* IN variables. */
domid_t domain;
u16 exec_domain;
- s32 cpu; /* -1 implies unpin */
+ u32 cpumap;
} dom0_pincpudomain_t;
/* Get trace buffers machine base address */
diff -urN b/xen/include/xen/sched.h c/xen/include/xen/sched.h
--- b/xen/include/xen/sched.h 2005-04-14 22:08:56.000000000 -0500
+++ c/xen/include/xen/sched.h 2005-04-15 11:52:18.000000000 -0500
@@ -58,6 +58,8 @@
void destroy_event_channels(struct domain *d);
int init_exec_domain_event_channels(struct exec_domain *ed);
+
+#define CPUMAP_RUNANYWHERE 0xFFFFFFFF
struct exec_domain
{
u32 processor;
@@ -83,6 +85,8 @@
atomic_t pausecnt;
+ u32 cpumap; /* which cpus this domain can run on */
+
struct arch_exec_domain arch;
};
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home