Hi,
We are working on reducing blackout time during domain migrations in xen
3.1.0. We ran into a problem when backend block driver removes the
backend device without waiting for the main kernel thread created by the
driver to actually complete all outstanding io. This happens in the
final stages of the migration process when guest domain is being
destroyed.
static int blkback_remove(struct xenbus_device *dev) {
struct backend_info *be = dev->dev.driver_data;
...
if (kthread_remove(be))
WPRINTK("BAD REMOVE REQUEST for %s\n", be->nodename);
...
backend_release(be);
return 0;
}
If "kthread" is still running and holds "be->refcnt",
"backend_release()" returns after
if (!atomic_dec_and_test(&be->refcnt))
return;
without reaching blkif_disconnect() to wait for io to complete.
At this point backend disappears from sysfs (and I assume, from
xenstore) radar, so there is no way to determine if block io has
actually completed on the source before "unleashing" migrated domain on
the target system.
We added a little more "plumbing" to wait for kthread to finish io
before calling "backend_release()". I am new to xen and would like to
ask if this is actually a "safe" thing to do. It looks like
"backend_remove()" is called by the xenwatch thread. I just want to make
sure that by hanging xenwatch thread at this point I am not creating
potential deadlock cases - in some disk io failure scenarios xenwatch
thread might get hung for a long period of time.
We also noticed that "unleashing" domain on the target is gated by the
destruction of the domain on the source. Destruction of domains with
lots of memory is a very time consuming process. To speed things up we
modified XendCheckpoint.py "save" function to do
dominfo._releaseDevices(force = True)
wait_for_outstanding_vbd_io_omplete() # "shutdown-done" in xenstore
unleash_the_target()
just before the current
dominfo.destroyDomain()
dominfo.testDeviceComplete()
Both changes - the driver and xend - work fine in our testing. Being new
to xen I just want to double check that I am not missing something. The
complete patch is below.
Thanks,
-Ed
diff -ur a/drivers/xen/blkback/xenbus.c b/drivers/xen/blkback/xenbus.c
--- a/drivers/xen/blkback/xenbus.c 2008-08-24 12:24:56.000000000
-0400
+++ b/drivers/xen/blkback/xenbus.c 2008-08-24 16:05:07.000000000
-0400
@@ -38,6 +38,7 @@
char *mode;
int group_added;
char *nodename;
+ wait_queue_head_t wq;
atomic_t refcnt;
pid_t kthread_pid;
int shutdown_signalled;
@@ -220,7 +221,6 @@
blkif->remove_requested = 1;
wake_up_process(blkif->xenblkd);
- blkif->xenblkd = NULL;
spin_unlock_irqrestore(&xenblkd_lock, flags);
@@ -296,6 +296,10 @@
if (kthread_remove(be))
WPRINTK("BAD REMOVE REQUEST for %s\n", be->nodename);
+ if (be->blkif) {
+ wait_event(be->wq, be->blkif->xenblkd == NULL);
+ }
+
down(&blkback_dev_sem);
xenvbd_sysfs_delif(dev);
dev->dev.driver_data = NULL;
@@ -311,9 +315,16 @@
*/
void blkback_close(blkif_t *blkif)
{
+ long flags;
+
blkif_disconnect(blkif);
vbd_sync(&blkif->vbd);
- blkif->remove_requested = 0;
+
+ spin_lock_irqsave(&xenblkd_lock, flags);
+ blkif->xenblkd = NULL;
+ blkif->remove_requested = 0;
+ wake_up(&blkif->be->wq);
+ spin_unlock_irqrestore(&xenblkd_lock, flags);
down(&blkback_dev_sem);
if (blkif->be->dev)
@@ -386,6 +397,8 @@
"allocating backend structure");
return -ENOMEM;
}
+
+ init_waitqueue_head(&be->wq);
be->dev = dev;
dev->dev.driver_data = be;
atomic_set(&be->refcnt, 1);
diff -ur ax/tools/python/xen/xend/XendCheckpoint.py
bx/tools/python/xen/xend/XendCheckpoint.py
--- ax/tools/python/xen/xend/XendCheckpoint.py 2008-08-25
17:18:14.000000000 -0400
+++ bx/tools/python/xen/xend/XendCheckpoint.py 2008-08-25
17:18:40.000000000 -0400
@@ -55,7 +55,7 @@
return buf
-def save(fd, dominfo, network, live, dst, checkpoint=False):
+def save(fd, dominfo, network, live, dst, checkpoint=False, sock=None):
write_exact(fd, SIGNATURE, "could not write guest state file:
signature")
config = sxp.to_string(dominfo.sxpr())
@@ -124,8 +124,17 @@
if checkpoint:
dominfo.resumeDomain()
else:
- dominfo.destroyDomain()
- dominfo.testDeviceComplete()
+ if sock is None:
+ dominfo.destroyDomain()
+ dominfo.testDeviceComplete()
+ else:
+ # handle migration - "Use the Force, Luke"
+ dominfo._releaseDevices(force = True)
+ dominfo.testDeviceComplete(io_done = True)
+ sock.shutdown(2)
+ dominfo.destroyDomain()
+ dominfo.testDeviceComplete()
+
try:
dominfo.setName(domain_name)
except VmError:
diff -ur ax/tools/python/xen/xend/XendDomain.py
bx/tools/python/xen/xend/XendDomain.py
--- ax/tools/python/xen/xend/XendDomain.py 2008-08-25
17:18:14.000000000 -0400
+++ bx/tools/python/xen/xend/XendDomain.py 2008-08-25
17:18:40.000000000 -0400
@@ -1242,7 +1242,8 @@
try:
sock.send("receive\n")
sock.recv(80)
- XendCheckpoint.save(sock.fileno(), dominfo, True, live,
dst)
+ XendCheckpoint.save(sock.fileno(), dominfo, True, live,
dst,
+ sock=sock)
finally:
sock.close()
diff -ur ax/tools/python/xen/xend/XendDomainInfo.py
bx/tools/python/xen/xend/XendDomainInfo.py
--- ax/tools/python/xen/xend/XendDomainInfo.py 2008-08-25
17:18:14.000000000 -0400
+++ bx/tools/python/xen/xend/XendDomainInfo.py 2008-08-25
17:18:40.000000000 -0400
@@ -1336,7 +1336,7 @@
if self.image:
self.image.createDeviceModel()
- def _releaseDevices(self, suspend = False):
+ def _releaseDevices(self, suspend = False, force = False):
"""Release all domain's devices. Nothrow guarantee."""
if suspend and self.image:
self.image.destroy(suspend)
@@ -1347,7 +1347,7 @@
for dev in t.list(devclass):
try:
log.debug("Removing %s", dev);
- self.destroyDevice(devclass, dev, False);
+ self.destroyDevice(devclass, dev, force)
except:
# Log and swallow any exceptions in removal --
# there's nothing more we can do.
@@ -1907,7 +1907,7 @@
except:
log.exception("Unwatching VM path failed.")
- def testDeviceComplete(self):
+ def testDeviceComplete(self, io_done = False):
""" For Block IO migration safety we must ensure that
the device has shutdown correctly, i.e. all blocks are
flushed to disk
@@ -1920,19 +1920,34 @@
test = 0
for i in ctrlr.deviceIDs():
be = "%s/%s" % (bep, i)
- try:
- hs = xstransact.Read(be,"hotplug-status")
- if hs is None:
- # dir must have been deleted and recreated
- log.info("Dev %s deleted (dir exists)", i)
- else:
- # Hmm.. device really is still there
- log.info("Dev %s still exists, looping", i)
- test = 1
- except:
- # exception reading directory - must be
- # gone now...
- log.info("Dev %s deleted", i)
+ if io_done:
+ try:
+ hs = xstransact.Read(be,"shutdown-done")
+ if hs is None:
+ # io is not completed yet
+ log.info("Dev %s io not done, looping", i)
+ test = 1
+ else:
+ #
+ log.info("Dev %s io is done", i)
+ except:
+ # exception reading directory - must be
+ # gone now...
+ log.info("Dev %s deleted", i)
+ else:
+ try:
+ hs = xstransact.Read(be,"hotplug-status")
+ if hs is None:
+ # dir must have been deleted and recreated
+ log.info("Dev %s deleted (dir exists)", i)
+ else:
+ # Hmm.. device really is still there
+ log.info("Dev %s still exists, looping", i)
+ test = 1
+ except:
+ # exception reading directory - must be
+ # gone now...
+ log.info("Dev %s deleted", i)
if test == 0:
break
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel
|
Home