WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-devel

[Xen-devel] [PATCH] 2nd try: 2/2 VCPU creation and allocation

Allocation map.

-- 
Ryan Harper
Software Engineer; Linux Technology Center
IBM Corp., Austin, Tx
(512) 838-9253   T/L: 678-9253
ryanh@xxxxxxxxxx


diffstat output:
 tools/examples/xmexample1               |    4 +
 tools/examples/xmexample2               |    3 +
 tools/python/xen/lowlevel/xc/xc.c       |    2 
 tools/python/xen/xend/XendDomainInfo.py |   16 ++++-
 tools/python/xen/xm/create.py           |    6 ++
 xen/common/dom0_ops.c                   |   90 +++++++++++++++++---------------
 xen/common/domain.c                     |   28 +++++++++
 xen/common/schedule.c                   |   11 ++-
 xen/include/public/dom0_ops.h           |    2 
 xen/include/xen/domain.h                |    2 
 xen/include/xen/sched.h                 |    2 
 11 files changed, 114 insertions(+), 52 deletions(-)

Signed-off-by: Ryan Harper <ryanh@xxxxxxxxxx>
---
diff -r 12d1e93653c2 tools/examples/xmexample1
--- a/tools/examples/xmexample1 Tue Oct 11 00:13:12 2005
+++ b/tools/examples/xmexample1 Mon Oct 10 19:35:55 2005
@@ -27,6 +27,10 @@
 
 # Number of Virtual CPUS to use, default is 1
 #vcpus = 1
+
+# A bitmap of which physical cpus are vcpus allowed to use.
+# ex1: 0x2 <-- bit 1 set means all vcpus will be created on CPU1
+#allocmap = 0xffffffff  # default value, vcpus can run on any cpu.
 
 #----------------------------------------------------------------------------
 # Define network interfaces.
diff -r 12d1e93653c2 tools/examples/xmexample2
--- a/tools/examples/xmexample2 Tue Oct 11 00:13:12 2005
+++ b/tools/examples/xmexample2 Mon Oct 10 19:35:55 2005
@@ -58,6 +58,9 @@
 # Number of Virtual CPUS to use, default is 1
 #vcpus = 1
 vcpus = 4 # make your domain a 4-way
+
+# A bitmap of which physical cpus are vcpus allowed to use.
+allocmap = 0x2  # start all of your VCPUs on CPU1
 
 #----------------------------------------------------------------------------
 # Define network interfaces.
diff -r 12d1e93653c2 tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c Tue Oct 11 00:13:12 2005
+++ b/tools/python/xen/lowlevel/xc/xc.c Mon Oct 10 19:35:55 2005
@@ -185,7 +185,7 @@
 
     static char *kwd_list[] = { "dom", "vcpu", "cpumap", NULL };
 
-    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|ii", kwd_list, 
+    if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i|iL", kwd_list, 
                                       &dom, &vcpu, &cpumap) )
         return NULL;
 
diff -r 12d1e93653c2 tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py   Tue Oct 11 00:13:12 2005
+++ b/tools/python/xen/xend/XendDomainInfo.py   Mon Oct 10 19:35:55 2005
@@ -266,6 +266,7 @@
     result['maxmem']    = get_cfg('maxmem',    int)
     result['maxmem_kb'] = get_cfg('maxmem_kb', int)
     result['cpu']       = get_cfg('cpu',       int)
+    result['allocmap']  = get_cfg('allocmap',  int)
     result['image']     = get_cfg('image')
 
     try:
@@ -438,6 +439,7 @@
             defaultInfo('cpu_weight',   lambda: 1.0)
             defaultInfo('vcpus',        lambda: 1)
             defaultInfo('vcpu_avail',   lambda: (1 << self.info['vcpus']) - 1)
+            defaultInfo('allocmap'  ,   lambda: None)
             defaultInfo('bootloader',   lambda: None)
             defaultInfo('backend',      lambda: [])
             defaultInfo('device',       lambda: [])
@@ -1018,6 +1020,16 @@
             self.image.handleBootloading()
 
         xc.domain_setcpuweight(self.domid, self.info['cpu_weight'])
+        cpu = self.info['cpu']
+        if cpu is not None and cpu != -1:
+            xc.domain_pincpu(self.domid, 0, 1 << cpu)
+
+        # set the domain alloc map for future vcpus,
+        # repin VCPU0 according to the alloc map
+        allocmap = self.info['allocmap']
+        if self.domid and allocmap:
+            xc.domain_pincpu(self.domid, -1, allocmap)  # domain allocmap
+            xc.domain_pincpu(self.domid, 0, allocmap)   # repin VCPU0
 
         # increase the total number of vcpus in domain
         xc.domain_vcpus_increase(self.domid, int(self.info['vcpus']));
@@ -1026,10 +1038,6 @@
         m = self.image.getDomainMemory(self.info['memory_KiB'])
         xc.domain_setmaxmem(self.domid, m)
         xc.domain_memory_increase_reservation(self.domid, m, 0, 0)
-
-        cpu = self.info['cpu']
-        if cpu is not None and cpu != -1:
-            xc.domain_pincpu(self.domid, 0, 1 << cpu)
 
         self.info['start_time'] = time.time()
 
diff -r 12d1e93653c2 tools/python/xen/xm/create.py
--- a/tools/python/xen/xm/create.py     Tue Oct 11 00:13:12 2005
+++ b/tools/python/xen/xm/create.py     Mon Oct 10 19:35:55 2005
@@ -151,6 +151,10 @@
 gopts.var('cpu', val='CPU',
           fn=set_int, default=None,
           use="CPU to run the domain on.")
+
+gopts.var('allocmap', val='ALLOCMAP',
+          fn=set_int, default=None,
+          use="Set default cpumap used for allocating vcpus.")
 
 gopts.var('vcpus', val='VCPUS',
           fn=set_int, default=1,
@@ -562,6 +566,8 @@
         config.append(['maxmem', vals.maxmem])
     if vals.cpu is not None:
         config.append(['cpu', vals.cpu])
+    if vals.allocmap is not None:
+        config.append(['allocmap', vals.allocmap])
     if vals.cpu_weight is not None:
         config.append(['cpu_weight', vals.cpu_weight])
     if vals.blkif:
diff -r 12d1e93653c2 xen/common/dom0_ops.c
--- a/xen/common/dom0_ops.c     Tue Oct 11 00:13:12 2005
+++ b/xen/common/dom0_ops.c     Mon Oct 10 19:35:55 2005
@@ -275,7 +275,7 @@
     {
         domid_t dom = op->u.pincpudomain.domain;
         struct domain *d = find_domain_by_id(dom);
-        struct vcpu *v;
+        struct vcpu *v = NULL;
         cpumap_t cpumap;
 
 
@@ -285,27 +285,33 @@
             break;
         }
         
-        if ( (op->u.pincpudomain.vcpu >= MAX_VIRT_CPUS) ||
-             !d->vcpu[op->u.pincpudomain.vcpu] )
-        {
-            ret = -EINVAL;
-            put_domain(d);
-            break;
-        }
-
-        v = d->vcpu[op->u.pincpudomain.vcpu];
-        if ( v == NULL )
-        {
-            ret = -ESRCH;
-            put_domain(d);
-            break;
-        }
-
-        if ( v == current )
-        {
-            ret = -EINVAL;
-            put_domain(d);
-            break;
+        /* don't bail on vcpu = -1 as that sets domain cpumap */
+        if ( (op->u.pincpudomain.vcpu != -1) && 
+              ((op->u.pincpudomain.vcpu >= MAX_VIRT_CPUS) ||
+              !d->vcpu[op->u.pincpudomain.vcpu])
+           )
+        {
+            ret = -EINVAL;
+            put_domain(d);
+            break;
+        }
+
+        /* don't get a struct vcpu pointer for -1 op */
+        if (op->u.pincpudomain.vcpu != -1) {
+           v = d->vcpu[op->u.pincpudomain.vcpu];
+           if ( v == NULL )
+           {
+               ret = -ESRCH;
+               put_domain(d);
+               break;
+           }
+
+           if ( v == current )
+           {
+               ret = -EINVAL;
+               put_domain(d);
+               break;
+           }
         }
 
         if ( copy_from_user(&cpumap, op->u.pincpudomain.cpumap,
@@ -316,24 +322,28 @@
             break;
         }
 
-        /* update cpumap for this vcpu */
-        v->cpumap = cpumap;
-
-        if ( cpumap == CPUMAP_RUNANYWHERE )
-        {
-            clear_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
-        }
-        else
-        {
-            /* pick a new cpu from the usable map */
-            int new_cpu = (int)find_first_set_bit(cpumap) % num_online_cpus();
-
-            vcpu_pause(v);
-            vcpu_migrate_cpu(v, new_cpu);
-            set_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
-            vcpu_unpause(v);
-        }
-
+        /* update domain vcpu alloc map */
+        if ( v == NULL ) {
+            d->allocmap = cpumap;
+        } else {
+            /* update cpumap for this vcpu */
+            v->cpumap = cpumap;
+
+           if ( cpumap == CPUMAP_RUNANYWHERE )
+           {
+               clear_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
+           }
+           else
+           {
+               /* pick a new cpu from the usable map */
+               int new_cpu = get_next_processor(d, v, &cpumap);
+
+               vcpu_pause(v);
+               vcpu_migrate_cpu(v, new_cpu);
+               set_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
+               vcpu_unpause(v);
+           }
+        }
         put_domain(d);
     }
     break;
diff -r 12d1e93653c2 xen/common/domain.c
--- a/xen/common/domain.c       Tue Oct 11 00:13:12 2005
+++ b/xen/common/domain.c       Mon Oct 10 19:35:55 2005
@@ -41,6 +41,7 @@
     atomic_set(&d->refcnt, 1);
     atomic_set(&v->pausecnt, 0);
 
+    d->allocmap  = CPUMAP_RUNANYWHERE;
     d->domain_id = dom_id;
     v->processor = cpu;
 
@@ -380,7 +381,7 @@
     v = d->vcpu[vcpuid];
 
     atomic_set(&v->pausecnt, 0);
-    v->cpumap = CPUMAP_RUNANYWHERE;
+    v->cpumap = d->allocmap;
 
     memcpy(&v->arch, &idle0_vcpu.arch, sizeof(v->arch));
 
@@ -469,6 +470,31 @@
 
     return -ENOSYS;
 }
+
+/* find the least loaded processor , ignorning vcpu v, in cpumap_t *map */
+int get_next_processor(struct domain* d, struct vcpu *v, cpumap_t *map) {
+    struct vcpu  *vc = NULL;
+    int pro, i, cnt[NR_CPUS] = { 0 };
+
+    /* count the processor layout for this dom, except for vcpu v 
+     * whose processor field may not have been set yet. */
+    for_each_vcpu( d, vc ) {
+        if (vc->vcpu_id != v->vcpu_id)
+            cnt[vc->processor]++;
+    }
+
+    /* start from the first allowable cpu, guard against bogus cpus */
+    pro = (int)find_first_set_bit(*map) % num_online_cpus();
+
+    /* pick least loaded processor in the map */
+    for ( i = pro; i < num_online_cpus(); i++ ) {
+        if ( test_bit(i, &*map) && (cnt[i] <= cnt[pro]) )
+            pro = i;
+    }
+
+    return pro;
+}
+
 
 /*
  * Local variables:
diff -r 12d1e93653c2 xen/common/schedule.c
--- a/xen/common/schedule.c     Tue Oct 11 00:13:12 2005
+++ b/xen/common/schedule.c     Mon Oct 10 19:35:55 2005
@@ -125,12 +125,13 @@
         v->next_in_list  = vc->next_in_list;
         vc->next_in_list = v;
 
-        if (test_bit(_VCPUF_cpu_pinned, &vc->vcpu_flags)) {
-            v->processor = (vc->processor + 1) % num_online_cpus();
+
+        /* XXX: if previous vcpu was pinned, mark new vcpu as pinned why? */
+        if (test_bit(_VCPUF_cpu_pinned, &vc->vcpu_flags))
             set_bit(_VCPUF_cpu_pinned, &v->vcpu_flags);
-        } else {
-            v->processor = (vc->processor + 1) % num_online_cpus();
-        }
+
+        v->processor = get_next_processor(d, v, &d->allocmap);
+
     }
 
     return v;
diff -r 12d1e93653c2 xen/include/public/dom0_ops.h
--- a/xen/include/public/dom0_ops.h     Tue Oct 11 00:13:12 2005
+++ b/xen/include/public/dom0_ops.h     Mon Oct 10 19:35:55 2005
@@ -181,7 +181,7 @@
 typedef struct {
     /* IN variables. */
     domid_t      domain;
-    u16          vcpu;
+    s16          vcpu;
     cpumap_t     *cpumap;
 } dom0_pincpudomain_t;
 
diff -r 12d1e93653c2 xen/include/xen/domain.h
--- a/xen/include/xen/domain.h  Tue Oct 11 00:13:12 2005
+++ b/xen/include/xen/domain.h  Mon Oct 10 19:35:55 2005
@@ -27,4 +27,6 @@
 
 extern void dump_pageframe_info(struct domain *d);
 
+int get_next_processor(struct domain *d, struct vcpu *v, cpumap_t *map);
+
 #endif /* __XEN_DOMAIN_H__ */
diff -r 12d1e93653c2 xen/include/xen/sched.h
--- a/xen/include/xen/sched.h   Tue Oct 11 00:13:12 2005
+++ b/xen/include/xen/sched.h   Mon Oct 10 19:35:55 2005
@@ -134,6 +134,8 @@
 
     /* Bitmask of CPUs which are holding onto this domain's state. */
     cpumask_t        cpumask;
+
+    cpumap_t         allocmap;        /* vcpu allocation bitmap */
 
     struct arch_domain arch;
 

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel