Re: [Xen-devel] trying to debug xen4 & qemu-dm, Xen reports "core dump failed"

["Justin T. Gibbs" <gibbs@xxxxxxxxxxx>]

On 1/26/2010 11:36 AM, 0bo0 wrote:
> in another thread @xen-devel, a crash @ DomU init under Xen 4.0.x, i
> was advised,
>
> On Mon, Jan 25, 2010 at 11:34 PM, Keir Fraser <keir.fraser@xxxxxxxxxxxxx> wrote:
>> If qemu-dm died without logging anything, it probably crashed again. If it
>> produced a core file you could load that in gdb and get a symbolic
>> backtrace.
>
> trying to turn on core dumps in xend-config, unfortunately, Xen seems
> to fail @ core dump ... in addition to failing to launch a DomU which
> works fine under Xen3,

...

I encountered this same problem.  I debugged it by pulling the OpenSuSE
source package for xen-4.0.0 and then manually rebuilding qemu with
symbols.  The problem is in a supplemental patch that OpenSuSE is adding
for CDROM media change support: cdrom-removable.patch.  This patched code
accesses an uninitialized variable, causing the segfault.

If you want to rebuild the package from source, I've attached a corrected
version of the cdrom-removable.patch file.  Just install the source rpm
(zypper si xen), replace the /usr/src/packages/SOURCES/cdrom-removable.patch
file with what is below, rebuild your packages (rpmbuild -ba
/usr/src/packages/SPECS/xen.spec), and install the packages found in
/usr/src/packages/RPMS/<arch>/.  You only need to reinstall xen-tools
to get the fix.

I've also posted information about this problem to the OpenSuSE-virtual
list.  Hopefully they will quickly incorporate a fix.

--
Justin

Index: xen-4.0.0-testing/tools/python/xen/xend/server/HalDaemon.py
===================================================================
--- /dev/null
+++ xen-4.0.0-testing/tools/python/xen/xend/server/HalDaemon.py
@@ -0,0 +1,243 @@
+#!/usr/bin/env python
+#  -*- mode: python; -*-
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2007 Pat Campbell <plc@xxxxxxxxxx>
+# Copyright (C) 2007 Novell Inc.
+#============================================================================
+
+"""hald (Hardware Abstraction Layer Daemon) watcher for Xen management
+   of removable block device media.
+
+"""
+
+import gobject
+import dbus
+import dbus.glib
+import os
+import types
+import sys
+import signal
+import traceback
+from xen.xend.xenstore.xstransact import xstransact, complete
+from xen.xend.xenstore.xsutil import xshandle
+from xen.xend import PrettyPrint
+from xen.xend import XendLogging
+from xen.xend.XendLogging import log
+
+DEVICE_TYPES = ['vbd', 'tap']
+
+class HalDaemon:
+    """The Hald block device watcher for XEN
+    """
+
+    """Default path to the log file. """
+    logfile_default = "/var/log/xen/hald.log"
+
+    """Default level of information to be logged."""
+    loglevel_default = 'INFO'
+
+
+    def __init__(self):
+
+        XendLogging.init(self.logfile_default, self.loglevel_default)
+        log.debug( "%s", "__init__")
+
+        self.udi_dict = {}
+        self.debug = 0
+        self.dbpath = "/local/domain/0/backend"
+        self.bus = dbus.SystemBus()
+        self.hal_manager_obj = self.bus.get_object('org.freedesktop.Hal', '/org/freedesktop/Hal/Manager')
+        self.hal_manager = dbus.Interface( self.hal_manager_obj, 'org.freedesktop.Hal.Manager')
+        self.gatherBlockDevices()
+        self.registerDeviceCallbacks()
+
+    def run(self):
+        log.debug( "%s", "In new run" );
+        try:
+            self.mainloop = gobject.MainLoop()
+            self.mainloop.run()
+        except KeyboardInterrupt, ex:
+            log.debug('Keyboard exception handler: %s', ex )
+            self.mainloop.quit()
+        except Exception, ex:
+            log.debug('Generic exception handler: %s', ex )
+            self.mainloop.quit()
+
+    def __del__(self):
+        log.debug( "%s", "In del " );
+        self.unRegisterDeviceCallbacks()
+        self.mainloop.quit()
+
+    def shutdown(self):
+        log.debug( "%s", "In shutdown now " );
+        self.unRegisterDeviceCallbacks()
+        self.mainloop.quit()
+
+    def stop(self):
+        log.debug( "%s", "In stop now " );
+        self.unRegisterDeviceCallbacks()
+        self.mainloop.quit()
+
+    def gatherBlockDevices(self):
+
+        # Get all the current devices from hal and save in a dictionary
+        try:
+            device_names = self.hal_manager.GetAllDevices()
+            i = 0;
+            for name in device_names:
+                #log.debug("device name, device=%s",name)
+               dev_obj = self.bus.get_object ('org.freedesktop.Hal', name)
+               dev = dbus.Interface (dev_obj, 'org.freedesktop.Hal.Device')
+               dev_properties = dev_obj.GetAllProperties(dbus_interface="org.freedesktop.Hal.Device")
+               if dev_properties.has_key('block.device'):
+                   dev_str = dev_properties['block.device']
+                   dev_major = dev_properties['block.major']
+                   dev_minor = dev_properties['block.minor']
+                   udi_info = {}
+                   udi_info['device'] = dev_str
+                   udi_info['major'] = dev_major
+                   udi_info['minor'] = dev_minor
+                   udi_info['udi'] = name
+                   self.udi_dict[i] = udi_info
+                   i = i + 1
+        except Exception, ex:
+            print >>sys.stderr, 'Exception gathering block devices:', ex
+            log.warn("Exception gathering block devices (%s)",ex)
+
+    #
+    def registerDeviceCallbacks(self):
+        # setup the callbacks for when the gdl changes
+        self.hal_manager.connect_to_signal('DeviceAdded', self.device_added_callback)
+        self.hal_manager.connect_to_signal('DeviceRemoved', self.device_removed_callback)
+
+    #
+    def unRegisterDeviceCallbacks(self):
+        # setup the callbacks for when the gdl changes
+        self.hal_manager.remove_signal_receiver(self.device_added_callback,'DeviceAdded')
+        self.hal_manager.remove_signal_receiver(self.device_removed_callback,'DeviceRemoved')
+
+    #
+    def device_removed_callback(self,udi):
+        log.debug('UDI %s was removed',udi)
+        self.show_dict(self.udi_dict)
+        for key in self.udi_dict:
+            udi_info = self.udi_dict[key]
+            if udi_info['udi'] == udi:
+                device = udi_info['device']
+                major = udi_info['major']
+                minor = udi_info['minor']
+                self.change_xenstore( "remove", device, major, minor)
+
+    # Adds device to dictionary if not already there
+    def device_added_callback(self,udi):
+        log.debug('UDI %s was added', udi)
+        self.show_dict(self.udi_dict)
+        dev_obj = self.bus.get_object ('org.freedesktop.Hal', udi)
+        dev = dbus.Interface (dev_obj, 'org.freedesktop.Hal.Device')
+        device = dev.GetProperty ('block.device')
+        major = dev.GetProperty ('block.major')
+        minor = dev.GetProperty ('block.minor')
+        udi_info = {}
+        udi_info['device'] = device
+        udi_info['major'] = major
+        udi_info['minor'] = minor
+        udi_info['udi'] = udi
+        already = 0
+        cnt = 0;
+        for key in self.udi_dict:
+            info = self.udi_dict[key]
+            if info['udi'] == udi:
+                already = 1
+                break
+            cnt = cnt + 1
+        if already == 0:
+           self.udi_dict[cnt] = udi_info;
+           log.debug('UDI %s was added, device:%s major:%s minor:%s index:%d\n', udi, device, major, minor, cnt)
+        self.change_xenstore( "add", device, major, minor)
+
+    # Debug helper, shows dictionary contents
+    def show_dict(self,dict=None):
+        if self.debug == 0 :
+            return
+        if dict == None :
+            dict = self.udi_dict
+        for key in dict:
+            log.debug('udi_info %s udi_info:%s',key,dict[key])
+
+    # Set or clear xenstore media-present depending on the action argument
+    #  for every vbd that has this block device
+    def change_xenstore(self,action, device, major, minor):
+        for type in DEVICE_TYPES:
+            path = self.dbpath + '/' + type
+            domains = xstransact.List(path)
+            log.debug('domains: %s', domains)
+            for domain in domains:   # for each domain
+                devices = xstransact.List( path + '/' + domain)
+                log.debug('devices: %s',devices)
+                for device in devices:  # for each vbd device
+                   str = device.split('/')
+                   vbd_type = None;
+                   vbd_physical_device = None
+                   vbd_media = None
+                   vbd_device_path = path + '/' + domain + '/' + device
+                   listing = xstransact.List(vbd_device_path)
+                   for entry in listing: # for each entry
+                       item = path + '/' + entry
+                       value = xstransact.Read( vbd_device_path + '/' + entry)
+                       log.debug('%s=%s',item,value)
+                       if item.find('media-present') != -1:
+                           vbd_media = item;
+                           vbd_media_path = item
+                       if item.find('physical-device') != -1:
+                           vbd_physical_device = value;
+                       if item.find('type') != -1:
+                           vbd_type = value;
+                   if vbd_type is not None and vbd_physical_device is not None and vbd_media is not None :
+                       inode = vbd_physical_device.split(':')
+                       imajor = parse_hex(inode[0])
+                       iminor = parse_hex(inode[1])
+                       log.debug("action:%s major:%s- minor:%s- imajor:%s- iminor:%s- inode: %s",
+                               action,major,minor, imajor, iminor, inode)
+                       if int(imajor) == int(major) and int(iminor) == int(minor):
+                           if action == "add":
+                               xs_dict = {'media': "1"}
+                               xstransact.Write(vbd_device_path, 'media-present', "1" )
+                               log.debug("wrote xenstore media-present 1 path:%s",vbd_media_path)
+                           else:
+                               xstransact.Write(vbd_device_path, 'media-present', "0" )
+                               log.debug("wrote xenstore media 0 path:%s",vbd_media_path)
+
+def mylog( fmt, *args):
+    f = open('/tmp/haldaemon.log', 'a')
+    print >>f, "HalDaemon ", fmt % args
+    f.close()
+
+
+def parse_hex(val):
+    try:
+        if isinstance(val, types.StringTypes):
+            return int(val, 16)
+        else:
+            return val
+    except ValueError:
+        return None
+
+if __name__ == "__main__":
+    watcher = HalDaemon()
+    watcher.run()
+    print 'Falling off end'
+
+
Index: xen-4.0.0-testing/tools/python/xen/xend/server/Hald.py
===================================================================
--- /dev/null
+++ xen-4.0.0-testing/tools/python/xen/xend/server/Hald.py
@@ -0,0 +1,125 @@
+#============================================================================
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of version 2.1 of the GNU Lesser General Public
+# License as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#============================================================================
+# Copyright (C) 2007 Pat Campbell <plc@xxxxxxxxxx>
+# Copyright (C) 2007 Novell Inc.
+#============================================================================
+
+import errno
+import types
+import os
+import sys
+import time
+import signal
+from traceback import print_exc
+
+from xen.xend.XendLogging import log
+
+class Hald:
+    def __init__(self):
+        self.ready = False
+        self.running = True
+
+    def run(self):
+        """Starts the HalDaemon process
+        """
+        self.ready = True
+        try:
+            myfile =  self.find("xen/xend/server/HalDaemon.py")
+            args = (["python", myfile ])
+            self.pid = self.daemonize("python", args )
+            #log.debug( "%s %s pid:%d", "Hald.py starting ", args, self.pid )
+        except:
+            self.pid = -1
+            log.debug("Unable to start HalDaemon process")
+
+    def shutdown(self):
+        """Shutdown the HalDaemon process
+        """
+        log.debug("%s  pid:%d", "Hald.shutdown()", self.pid)
+        self.running = False
+        self.ready = False
+        if self.pid != -1:
+            try:
+                os.kill(self.pid, signal.SIGINT)
+            except:
+                print_exc()
+
+    def daemonize(self,prog, args):
+        """Runs a program as a daemon with the list of arguments.  Returns the PID
+        of the daemonized program, or returns 0 on error.
+        Copied from xm/create.py instead of importing to reduce coupling
+        """
+        r, w = os.pipe()
+        pid = os.fork()
+
+        if pid == 0:
+            os.close(r)
+            w = os.fdopen(w, 'w')
+            os.setsid()
+            try:
+                pid2 = os.fork()
+            except:
+                pid2 = None
+            if pid2 == 0:
+                os.chdir("/")
+                env = os.environ.copy()
+                env['PYTHONPATH'] = self.getpythonpath()
+                for fd in range(0, 256):
+                    try:
+                        os.close(fd)
+                    except:
+                        pass
+                os.open("/dev/null", os.O_RDWR)
+                os.dup2(0, 1)
+                os.dup2(0, 2)
+                os.execvpe(prog, args, env)
+                os._exit(1)
+            else:
+                w.write(str(pid2 or 0))
+                w.close()
+                os._exit(0)
+        os.close(w)
+        r = os.fdopen(r)
+        daemon_pid = int(r.read())
+        r.close()
+        os.waitpid(pid, 0)
+        #log.debug( "daemon_pid: %d", daemon_pid )
+        return daemon_pid
+
+    def getpythonpath(self):
+        str = " "
+        for p in sys.path:
+            if str != " ":
+                str = str + ":" + p
+            else:
+                if str != "":
+                   str = p
+        return str
+
+    def find(self,path, matchFunc=os.path.isfile):
+        """Find a module in the sys.path
+        From web page: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52224
+        """
+        for dirname in sys.path:
+            candidate = os.path.join(dirname, path)
+            if matchFunc(candidate):
+                return candidate
+        raise Error("Can't find file %s" % path)
+
+if __name__ == "__main__":
+    watcher = Hald()
+    watcher.run()
+    time.sleep(10)
+    watcher.shutdown()
Index: xen-4.0.0-testing/tools/python/xen/xend/server/SrvServer.py
===================================================================
--- xen-4.0.0-testing.orig/tools/python/xen/xend/server/SrvServer.py
+++ xen-4.0.0-testing/tools/python/xen/xend/server/SrvServer.py
@@ -56,6 +56,7 @@ from xen.web.SrvDir import SrvDir
 
 from SrvRoot import SrvRoot
 from XMLRPCServer import XMLRPCServer
+from xen.xend.server.Hald import Hald
 
 xoptions = XendOptions.instance()
 
@@ -245,6 +246,8 @@ def _loadConfig(servers, root, reload):
     if xoptions.get_xend_unix_xmlrpc_server():
         servers.add(XMLRPCServer(XendAPI.AUTH_PAM, False))
 
+    servers.add(Hald())
+
 
 def create():
     root = SrvDir()
Index: xen-4.0.0-testing/tools/ioemu-remote/xenstore.c
===================================================================
--- xen-4.0.0-testing.orig/tools/ioemu-remote/xenstore.c
+++ xen-4.0.0-testing/tools/ioemu-remote/xenstore.c
@@ -594,6 +594,20 @@
 #endif
 
         bs = bdrv_new(dev);
+        /* if cdrom pyhsical put a watch on media-present */
+        if (bdrv_get_type_hint(bs) ==  BDRV_TYPE_CDROM) {
+            if (drv && !strcmp(drv, "phy")) {
+                if (pasprintf(&buf, "%s/media-present", bpath) != -1) {
+                    if (bdrv_is_inserted(bs))
+                        xs_write(xsh, XBT_NULL, buf, "1", strlen("1"));
+                    else {
+                        xs_write(xsh, XBT_NULL, buf, "0", strlen("0"));
+                    }
+                    xs_watch(xsh, buf, "media-present");
+                }
+            }
+        }
+
         /* check if it is a cdrom */
         if (danger_type && !strcmp(danger_type, "cdrom")) {
             bdrv_set_type_hint(bs, BDRV_TYPE_CDROM);
@@ -938,6 +951,50 @@ void xenstore_record_dm_state(const char
     xenstore_record_dm("state", state);
 }
 
+void xenstore_process_media_change_event(char **vec)
+{
+    char *media_present = NULL;
+    unsigned int len;
+
+    media_present = xs_read(xsh, XBT_NULL, vec[XS_WATCH_PATH], &len);
+
+    if (media_present) {
+        BlockDriverState *bs;
+        char *buf = NULL, *cp = NULL, *path = NULL, *dev = NULL;
+
+        path = strdup(vec[XS_WATCH_PATH]);
+        cp = strstr(path, "media-present");
+        if (cp){
+            *(cp-1) = '\0';
+            pasprintf(&buf, "%s/dev", path);
+            dev = xs_read(xsh, XBT_NULL, buf, &len);
+            if (dev) {
+                if ( !strncmp(dev, "xvd", 3)) {
+                    memmove(dev, dev+1, strlen(dev));
+                    dev[0] = 'h';
+                    dev[1] = 'd';
+                }
+                bs = bdrv_find(dev);
+                if (!bs) {
+                    term_printf("device not found\n");
+                    return;
+                }
+                if (strcmp(media_present, "0") == 0 && bs) {
+                    bdrv_close(bs);
+                }
+                else if (strcmp(media_present, "1") == 0 &&
+                        bs != NULL && bs->drv == NULL) {
+                    if (bdrv_open(bs, bs->filename, 0 /* snapshot */) < 0) {
+                        fprintf(logfile, "%s() qemu: could not open cdrom disk '%s'\n",
+                                __func__, bs->filename);
+                    }
+                    bs->media_changed = 1;
+                }
+            }
+        }
+    }
+}
+
 void xenstore_process_event(void *opaque)
 {
     char **vec, *offset, *bpath = NULL, *buf = NULL, *drv = NULL, *image = NULL;
@@ -968,6 +1025,11 @@ void xenstore_process_event(void *opaque
             xenstore_watch_callbacks[i].cb(vec[XS_WATCH_TOKEN],
                                            xenstore_watch_callbacks[i].opaque);
 
+    if (!strcmp(vec[XS_WATCH_TOKEN], "media-present")) {
+        xenstore_process_media_change_event(vec);
+        goto out;
+    }
+
     if (strncmp(vec[XS_WATCH_TOKEN], "hd", 2) ||
         strlen(vec[XS_WATCH_TOKEN]) != 3)
         goto out;

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel