WARNING - OLD ARCHIVES

This is an archived copy of the Xen.org mailing list, which we have preserved to ensure that existing links to archives are not broken. The live archive, which contains the latest emails, can be found at http://lists.xen.org/
   
 
 
Xen 
 
Home Products Support Community News
 
   
 

xen-devel

[Xen-devel] NUMA node migration

To: xen-devel@xxxxxxxxxxxxxxxxxxx
Subject: [Xen-devel] NUMA node migration
From: "Andre Przywara" <andre.przywara@xxxxxxx>
Date: Fri, 21 Dec 2007 17:42:50 +0100
Delivery-date: Fri, 21 Dec 2007 08:46:21 -0800
Envelope-to: www-data@xxxxxxxxxxxxxxxxxx
List-help: <mailto:xen-devel-request@lists.xensource.com?subject=help>
List-id: Xen developer discussion <xen-devel.lists.xensource.com>
List-post: <mailto:xen-devel@lists.xensource.com>
List-subscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=subscribe>
List-unsubscribe: <http://lists.xensource.com/cgi-bin/mailman/listinfo/xen-devel>, <mailto:xen-devel-request@lists.xensource.com?subject=unsubscribe>
Sender: xen-devel-bounces@xxxxxxxxxxxxxxxxxxx
User-agent: Thunderbird 2.0.0.6 (X11/20070728)
Hi,

the following patch adds NUMA node migration based on live migration to xend. By adding another parameter to "xm migrate" the target NUMA node number gets propagated to the target host (can be both localhost or a remote host). The restore function then sets the VCPU affinity accordingly. Only changes Python code in xend. I hope that the patch doesn't break XenAPI compatibility (adding a parameter seems fine?).

# xm migrate --live --node=<nodenr> <domid> localhost
<nodenr> is the number as shown with 'xm info' under node_to_cpu

I am aware that using live migration isn't the best approach (takes twice the memory and quite some time), but it's less intrusive and works fine (given localhost migration stability...)

Feedback appreciated, especially since I speak Python since Monday...

Regards,
Andre.

--
Andre Przywara
AMD-Operating System Research Center (OSRC), Dresden, Germany
Tel: +49 351 277-84917
----to satisfy European Law for business letters:
AMD Saxony Limited Liability Company & Co. KG,
Wilschdorfer Landstr. 101, 01109 Dresden, Germany
Register Court Dresden: HRA 4896, General Partner authorized
to represent: AMD Saxony LLC (Wilmington, Delaware, US)
General Manager of AMD Saxony LLC: Dr. Hans-R. Deppe, Thomas McCoy
diff -r 1f4b29eaf7f4 -r 1a9f3e26552d tools/python/xen/xend/XendAPI.py
--- a/tools/python/xen/xend/XendAPI.py  Thu Dec 20 17:30:27 2007 +0000
+++ b/tools/python/xen/xend/XendAPI.py  Fri Dec 21 17:34:38 2007 +0100
@@ -1761,9 +1761,10 @@ class XendAPI(object):
 
         resource = other_config.get("resource", 0)
         port = other_config.get("port", 0)
+        node = other_config.get("node", 0)
         
         xendom.domain_migrate(xeninfo.getDomid(), destination_url,
-                              bool(live), resource, port)
+                              bool(live), resource, port, node)
         return xen_api_success_void()
 
     def VM_save(self, _, vm_ref, dest, checkpoint):
diff -r 1f4b29eaf7f4 -r 1a9f3e26552d tools/python/xen/xend/XendCheckpoint.py
--- a/tools/python/xen/xend/XendCheckpoint.py   Thu Dec 20 17:30:27 2007 +0000
+++ b/tools/python/xen/xend/XendCheckpoint.py   Fri Dec 21 17:34:38 2007 +0100
@@ -22,6 +22,7 @@ from xen.xend.XendLogging import log
 from xen.xend.XendLogging import log
 from xen.xend.XendConfig import XendConfig
 from xen.xend.XendConstants import *
+from xen.xend import XendNode
 
 SIGNATURE = "LinuxGuestRecord"
 QEMU_SIGNATURE = "QemuDeviceModelRecord"
@@ -56,10 +57,23 @@ def read_exact(fd, size, errmsg):
     return buf
 
 
-def save(fd, dominfo, network, live, dst, checkpoint=False):
+def insert_after(list, pred, value):
+    for i,k in enumerate(list):
+        if type(k) == type([]):
+           if k[0] == pred:
+              list.insert (i+1, value)
+    return
+
+
+def save(fd, dominfo, network, live, dst, checkpoint=False, node=-1):
     write_exact(fd, SIGNATURE, "could not write guest state file: signature")
 
-    config = sxp.to_string(dominfo.sxpr())
+    sxprep = dominfo.sxpr()
+
+    if node > -1:
+        insert_after(sxprep,'vcpus',['node', str(node)])
+
+    config = sxp.to_string(sxprep)
 
     domain_name = dominfo.getName()
     # Rename the domain temporarily, so that we don't get a name clash if this
@@ -175,6 +189,21 @@ def restore(xd, fd, dominfo = None, paus
         dominfo.resume()
     else:
         dominfo = xd.restore_(vmconfig)
+
+    # repin domain vcpus if a target node number was specified 
+    # this is done prior to memory allocation to aide in memory
+    # distribution for NUMA systems.
+    nodenr = -1
+    for i,l in enumerate(vmconfig):
+        if type(l) == type([]):
+            if l[0] == 'node':
+                nodenr = int(l[1])
+
+    if nodenr >= 0:
+        node_to_cpu = XendNode.instance().xc.physinfo()['node_to_cpu']
+        if nodenr < len(node_to_cpu):
+            for v in range(0, dominfo.info['VCPUs_max']):
+                 xc.vcpu_setaffinity(dominfo.domid, v, node_to_cpu[nodenr])
 
     store_port   = dominfo.getStorePort()
     console_port = dominfo.getConsolePort()
diff -r 1f4b29eaf7f4 -r 1a9f3e26552d tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py       Thu Dec 20 17:30:27 2007 +0000
+++ b/tools/python/xen/xend/XendDomain.py       Fri Dec 21 17:34:38 2007 +0100
@@ -1255,7 +1255,7 @@ class XendDomain:
 
         return val       
 
-    def domain_migrate(self, domid, dst, live=False, resource=0, port=0):
+    def domain_migrate(self, domid, dst, live=False, resource=0, port=0, 
node=-1):
         """Start domain migration.
         
         @param domid: Domain ID or Name
@@ -1268,6 +1268,8 @@ class XendDomain:
         @type live: bool
         @keyword resource: not used??
         @rtype: None
+        @keyword node: use node number for target
+        @rtype: int 
         @raise XendError: Failed to migrate
         @raise XendInvalidDomain: Domain is not valid        
         """
@@ -1296,7 +1298,7 @@ class XendDomain:
 
         sock.send("receive\n")
         sock.recv(80)
-        XendCheckpoint.save(sock.fileno(), dominfo, True, live, dst)
+        XendCheckpoint.save(sock.fileno(), dominfo, True, live, dst, node=node)
         sock.close()
 
     def domain_save(self, domid, dst, checkpoint=False):
diff -r 1f4b29eaf7f4 -r 1a9f3e26552d tools/python/xen/xm/migrate.py
--- a/tools/python/xen/xm/migrate.py    Thu Dec 20 17:30:27 2007 +0000
+++ b/tools/python/xen/xm/migrate.py    Fri Dec 21 17:34:38 2007 +0100
@@ -43,6 +43,10 @@ gopts.opt('port', short='p', val='portnu
           fn=set_int, default=0,
           use="Use specified port for migration.")
 
+gopts.opt('node', short='n', val='nodenum',
+          fn=set_int, default=-1,
+          use="Use specified NUMA node on target.")
+
 gopts.opt('resource', short='r', val='MBIT',
           fn=set_int, default=0,
           use="Set level of resource usage for migration.")
@@ -65,11 +69,13 @@ def main(argv):
         vm_ref = get_single_vm(dom)
         other_config = {
             "port":     opts.vals.port,
-            "resource": opts.vals.resource
+            "resource": opts.vals.resource,
+            "node":     opts.vals.node
             }
         server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live),
                                  other_config)
     else:
         server.xend.domain.migrate(dom, dst, opts.vals.live,
                                    opts.vals.resource,
-                                   opts.vals.port)
+                                   opts.vals.port,
+                                   opts.vals.node)
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
<Prev in Thread] Current Thread [Next in Thread>
  • [Xen-devel] NUMA node migration, Andre Przywara <=